Upstream Intel NPU Plugin (#23537)
### Details: - Add ENABLE_INTEL_NPU option - Add ENABLE_NPU_DEBUG_CAPS option - Add npu plugin codes to src/plugins/intel_npu/ - Add src/inference/include/openvino/runtime/intel_npu/properties.hpp - Add thirdparty level-zero and level-zero-ext to src/plugins/intel_npu/thirdparty - Export openvino::openvino_npu_al and openvino::openvino_npu_logger_utils ### Tickets: - E-115936 --------- Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>
This commit is contained in:
Xin Wang
GitHub
parent
1c82c171bf
commit
8463eee2f6
|
|
@ -100,6 +100,8 @@ jobs:
|
|||
if [[ -f "${INSTALL_DIR}/setupvars.sh" ]]; then
|
||||
source ${INSTALL_DIR}/setupvars.sh
|
||||
fi
|
||||
# Needed as ze_loader.so is under INSTALL_TEST_DIR
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${INSTALL_TEST_DIR}
|
||||
|
||||
python3 ${PARALLEL_TEST_SCRIPT} -e ${INSTALL_TEST_DIR}/ov_cpu_func_tests -c ${PARALLEL_TEST_CACHE} -w ${INSTALL_TEST_DIR} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke*
|
||||
timeout-minutes: 25
|
||||
|
|
|
|||
|
|
@ -212,6 +212,7 @@ jobs:
|
|||
tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz \
|
||||
tests/ov_cpu_func_tests \
|
||||
tests/libopenvino_template_extension.so \
|
||||
tests/libze_loader.so* \
|
||||
tests/functional_test_utils/layer_tests_summary/*
|
||||
popd
|
||||
|
||||
|
|
|
|||
|
|
@ -198,13 +198,13 @@ jobs:
|
|||
- name: Ctest - OpenVINO unit tests
|
||||
shell: cmd
|
||||
run: |
|
||||
set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin
|
||||
set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin;${{ env.BUILD_DIR }}\bin\Release
|
||||
ctest -C ${{ env.CMAKE_BUILD_TYPE }} --test-dir ${{ env.BUILD_DIR }} -V -L UNIT
|
||||
|
||||
- name: Perform code tracing via ITT collector
|
||||
shell: cmd
|
||||
run: |
|
||||
set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin
|
||||
set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin;${{ env.BUILD_DIR }}\bin\Release
|
||||
|
||||
python3 ${{ env.OPENVINO_REPO }}\thirdparty\itt_collector\runtool\sea_runtool.py ^
|
||||
--bindir ${{ env.OPENVINO_REPO }}\bin\intel64\${{ env.CMAKE_BUILD_TYPE }} ^
|
||||
|
|
@ -233,7 +233,7 @@ jobs:
|
|||
Compress-Archive @compress
|
||||
|
||||
$compress = @{
|
||||
Path = "${{ env.OPENVINO_REPO }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }}/ov_cpu_func_tests.exe", "${{ env.OPENVINO_REPO }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }}/openvino_template_extension.dll", "${{ env.OPENVINO_REPO }}/src/tests/test_utils/functional_test_utils/layer_tests_summary", "${{ env.INSTALL_DIR }}/runtime/3rdparty/tbb"
|
||||
Path = "${{ env.OPENVINO_REPO }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }}/ov_cpu_func_tests.exe", "${{ env.BUILD_DIR }}/bin/Release/ze_loader.dll", "${{ env.OPENVINO_REPO }}/bin/intel64/${{ env.CMAKE_BUILD_TYPE }}/openvino_template_extension.dll", "${{ env.OPENVINO_REPO }}/src/tests/test_utils/functional_test_utils/layer_tests_summary", "${{ env.INSTALL_DIR }}/runtime/3rdparty/tbb"
|
||||
CompressionLevel = "Optimal"
|
||||
DestinationPath = "${{ env.BUILD_DIR }}/openvino_tests.zip"
|
||||
}
|
||||
|
|
@ -356,7 +356,7 @@ jobs:
|
|||
- name: Run with CC-ed runtime
|
||||
shell: cmd
|
||||
run: |
|
||||
set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin
|
||||
set path=%path%;${{ env.OPENVINO_REPO }}\temp\tbb\bin;${{ env.BUILD_DIR }}\bin\Release
|
||||
${{ env.OPENVINO_REPO }}\bin\intel64\${{ env.CMAKE_BUILD_TYPE }}\benchmark_app.exe -niter 1 -nireq 1 -m ${{ env.MODELS_PATH }}\models\test_model\test_model_fp32.xml -d CPU
|
||||
|
||||
CPU_Functional_Tests:
|
||||
|
|
@ -414,7 +414,7 @@ jobs:
|
|||
- name: Intel CPU plugin func tests (parallel)
|
||||
shell: cmd
|
||||
run: |
|
||||
set path=%path%;${{ env.INSTALL_TEST_DIR }}\tbb\bin;${{ env.INSTALL_TEST_DIR }}\tbb
|
||||
set path=%path%;${{ env.INSTALL_TEST_DIR }}\tbb\bin;${{ env.INSTALL_TEST_DIR }}\tbb;${{ env.INSTALL_TEST_DIR }}
|
||||
python3 ${{ env.PARALLEL_TEST_SCRIPT }} -e ${{ env.INSTALL_TEST_DIR }}\ov_cpu_func_tests.exe -w ${{ env.INSTALL_TEST_DIR }} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke*
|
||||
timeout-minutes: 60
|
||||
|
||||
|
|
|
|||
|
|
@ -72,3 +72,9 @@
|
|||
[submodule "src/plugins/intel_cpu/thirdparty/mlas"]
|
||||
path = src/plugins/intel_cpu/thirdparty/mlas
|
||||
url = https://github.com/openvinotoolkit/mlas.git
|
||||
[submodule "src/plugins/intel_npu/thirdparty/level-zero"]
|
||||
path = src/plugins/intel_npu/thirdparty/level-zero
|
||||
url = https://github.com/oneapi-src/level-zero.git
|
||||
[submodule "src/plugins/intel_npu/thirdparty/level-zero-ext"]
|
||||
path = src/plugins/intel_npu/thirdparty/level-zero-ext
|
||||
url = https://github.com/intel/level-zero-npu-extensions.git
|
||||
|
|
|
|||
|
|
@ -44,7 +44,16 @@ endif()
|
|||
|
||||
ov_dependent_option (ENABLE_ONEDNN_FOR_GPU "Enable oneDNN with GPU support" ${ENABLE_ONEDNN_FOR_GPU_DEFAULT} "ENABLE_INTEL_GPU" OFF)
|
||||
|
||||
if(X86_64)
|
||||
set(ENABLE_INTEL_NPU_DEFAULT ON)
|
||||
else()
|
||||
set(ENABLE_INTEL_NPU_DEFAULT OFF)
|
||||
endif()
|
||||
|
||||
ov_dependent_option (ENABLE_INTEL_NPU "NPU plugin for OpenVINO runtime" ${ENABLE_INTEL_NPU_DEFAULT} "X86 OR X86_64;NOT APPLE" OFF)
|
||||
|
||||
ov_option (ENABLE_DEBUG_CAPS "enable OpenVINO debug capabilities at runtime" OFF)
|
||||
ov_dependent_option (ENABLE_NPU_DEBUG_CAPS "enable NPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_NPU" OFF)
|
||||
ov_dependent_option (ENABLE_GPU_DEBUG_CAPS "enable GPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_GPU" OFF)
|
||||
ov_dependent_option (ENABLE_CPU_DEBUG_CAPS "enable CPU debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS;ENABLE_INTEL_CPU" OFF)
|
||||
ov_dependent_option (ENABLE_SNIPPETS_DEBUG_CAPS "enable Snippets debug capabilities at runtime" ON "ENABLE_DEBUG_CAPS" OFF)
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ macro(ov_cpack_settings)
|
|||
endif()
|
||||
|
||||
# intel-npu
|
||||
if(ENABLE_INTEL_NPU OR BUILD_npu OR BUILD_vpux-plugin OR BUILD_applications.ai.vpu-accelerators.vpux-plugin)
|
||||
if(ENABLE_INTEL_NPU)
|
||||
set(CPACK_COMPONENT_NPU_DESCRIPTION "Intel® Neural Processing Unit inference plugin")
|
||||
set(CPACK_COMPONENT_NPU_DEPENDS "${OV_CPACK_COMP_CORE}")
|
||||
set(CPACK_DEBIAN_NPU_PACKAGE_NAME "libopenvino-intel-npu-plugin-${cpack_name_ver}")
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@ macro(ov_cpack_settings)
|
|||
(NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE}_python.*" OR ENABLE_PYTHON_PACKAGING) AND
|
||||
# temporary block nvidia
|
||||
NOT item STREQUAL "nvidia" AND
|
||||
# temporary block npu
|
||||
NOT item STREQUAL "npu" AND
|
||||
# don't install Intel OpenMP
|
||||
NOT item STREQUAL "omp" AND
|
||||
# the same for pugixml
|
||||
|
|
@ -179,7 +181,7 @@ macro(ov_cpack_settings)
|
|||
endif()
|
||||
|
||||
# intel-npu
|
||||
if(ENABLE_INTEL_NPU OR BUILD_npu OR BUILD_vpux-plugin OR BUILD_applications.ai.vpu-accelerators.vpux-plugin)
|
||||
if(ENABLE_INTEL_NPU AND "npu" IN_LIST CPACK_COMPONENTS_ALL)
|
||||
set(CPACK_COMPONENT_NPU_DESCRIPTION "Intel® Neural Processing Unit inference plugin")
|
||||
set(CPACK_RPM_NPU_PACKAGE_REQUIRES "${core_package}")
|
||||
set(CPACK_RPM_NPU_PACKAGE_NAME "libopenvino-intel-npu-plugin-${cpack_name_ver}")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
// Copyright (C) 2018-2023 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
/**
|
||||
* @brief A header for advanced hardware related properties for NPU plugin
|
||||
* To use in set_property, compile_model, import_model, get_property methods
|
||||
*
|
||||
* @file openvino/runtime/intel_npu/properties.hpp
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "openvino/runtime/properties.hpp"
|
||||
|
||||
namespace ov {
|
||||
|
||||
/**
|
||||
* @defgroup ov_runtime_npu_prop_cpp_api Intel NPU specific properties
|
||||
* @ingroup ov_runtime_cpp_api
|
||||
* Set of Intel NPU specific properties.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Namespace with Intel NPU specific properties
|
||||
*/
|
||||
namespace intel_npu {
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU plugin]
|
||||
* Type: uint64_t
|
||||
* Read-only property to get size of already allocated NPU DDR memory (both for discrete/integrated NPU devices)
|
||||
*
|
||||
* Note: Queries driver both for discrete/integrated NPU devices
|
||||
* @ingroup ov_runtime_npu_prop_cpp_api
|
||||
*/
|
||||
static constexpr ov::Property<uint64_t, ov::PropertyMutability::RO> device_alloc_mem_size{"NPU_DEVICE_ALLOC_MEM_SIZE"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU plugin]
|
||||
* Type: uint64_t
|
||||
* Read-only property to get size of available NPU DDR memory (both for discrete/integrated NPU devices)
|
||||
*
|
||||
* Note: Queries driver both for discrete/integrated NPU devices
|
||||
* @ingroup ov_runtime_npu_prop_cpp_api
|
||||
*/
|
||||
static constexpr ov::Property<uint64_t, ov::PropertyMutability::RO> device_total_mem_size{"NPU_DEVICE_TOTAL_MEM_SIZE"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU plugin]
|
||||
* Type: uint32_t
|
||||
* Read-only property to get NPU driver version (for both discrete/integrated NPU devices)
|
||||
* @ingroup ov_runtime_npu_prop_cpp_api
|
||||
*/
|
||||
static constexpr ov::Property<uint32_t, ov::PropertyMutability::RO> driver_version{"NPU_DRIVER_VERSION"};
|
||||
|
||||
} // namespace intel_npu
|
||||
} // namespace ov
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
if (NOT ENABLE_INTEL_NPU)
|
||||
return()
|
||||
endif()
|
||||
|
||||
#
|
||||
# Build properties
|
||||
#
|
||||
|
||||
set(NPU_DEVICE_NAME "NPU")
|
||||
string(TOLOWER "${NPU_DEVICE_NAME}" NPU_PLUGIN_COMPONENT)
|
||||
set(NPU_INTERNAL_COMPONENT "${NPU_PLUGIN_COMPONENT}_internal")
|
||||
|
||||
set(NPU_PLUGIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
include(cmake/features.cmake)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
|
||||
if(ENABLE_NPU_DEBUG_CAPS)
|
||||
add_compile_definitions(NPU_PLUGIN_DEVELOPER_BUILD)
|
||||
endif()
|
||||
|
||||
add_subdirectory(thirdparty EXCLUDE_FROM_ALL)
|
||||
|
||||
add_subdirectory(src)
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
# NPU Plugin
|
||||
|
||||
## Introduction
|
||||
|
||||
This is the OpenVINO Plugin for Intel® Neural Processing Unit (NPU) devices.
|
||||
|
||||
|
||||
## Supported Platforms
|
||||
|
||||
OpenVINO™ toolkit is officially supported and validated on the following platforms:
|
||||
|
||||
| Host | NPU device | OS (64-bit) |
|
||||
| :--- | :--- | :--- |
|
||||
| Raptor Lake (discrete NPU) | NPU 3700 | MS Windows* 11 |
|
||||
| Meteor Lake (integrated NPU) | NPU 3720 | Ubuntu* 22, MS Windows* 11 |
|
||||
|
||||
|
||||
|
||||
## High Level Design
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Description
|
||||
|
||||
NPU Plugin is a software library that:
|
||||
* Implements the unified OpenVINO Plugin API used to compile and execute neural networks on NPU devices.
|
||||
* Uses the graph extension API exposed by the driver to convert the OpenVINO specific representation of the model into a proprietary format. The compiler performs platform specific optimizations in order to efficiently schedule the execution of layers and memory transactions on various NPU hardware submodules.
|
||||
* Uses the Level Zero API implemented by the NPU user mode driver (UMD) to execute the model on the device.
|
||||
|
||||
The plugin library is included inside the OpenVINO package while the compiler is packaged inside UMD and released separately.
|
||||
|
||||
Note: Aligning with the platform and OpenVINO documentation, neural networks will be referred to with the more generic term of models in the rest of this document.
|
||||
|
||||
|
||||
## Model Compilation
|
||||
|
||||
NPU plugin implements the OpenVINO Core "compile_model" API that converts the model representation into a proprietary format that can be executed on the NPU device:
|
||||
|
||||
```
|
||||
ov::CompiledModel compiled_model = core.compile_model(model, "NPU" [, config]);
|
||||
```
|
||||
|
||||
### Model caching
|
||||
|
||||
There are two important compilation related metrics when executing models on NPU devices:
|
||||
* First Ever Inference Latency (FEIL): Measures all steps required to compile and execute a model on the device for the first time. It includes model compilation time, the time required to load and initialize the model on the device and the first inference execution.
|
||||
* First Inference Latency (FIL): Measures the time required to load and initialize the pre-compiled model on the device and the first inference execution.
|
||||
|
||||
|
||||
#### UMD dynamic model caching
|
||||
|
||||
UMD model caching is enabled by default in the current NPU driver to improve time to first inference (FIL). The model is stored in the cache after the compilation (included in FEIL) based on a hash key. The UMD generates the key from the input IR model and build arguments and then requests the DirectX Shader cache session to store the model with the computed key. Any subsequent request to compile the same IR model with the same arguments would cause the pre-compiled model to be read from the cache instead of being recompiled.
|
||||
|
||||
#### OpenVINO model caching
|
||||
|
||||
It is enabled when `ov::cache_dir` property is set and it is a common mechanism for all OpenVINO plugins. UMD model caching will be automatically bypassed by the NPU plugin when `ov::cache_dir` is set so the model will only be stored in the OpenVINO cache after the compilation. When a cache hit occurs for subsequent compilation requests, plugin will import the model instead of recompiling it.
|
||||
|
||||
More details about OpenVINO model caching can be found here: [Model Caching Overview](https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Model_caching_overview.html).
|
||||
|
||||
### Compiler adapters
|
||||
|
||||
Two additional layers are required to support the compiler from driver:
|
||||
* Compiler Adapter - It serializes the OpenVINO internal representation of the model (ov::model) into an in-memory IR that will be provided to the NPU driver
|
||||
* VCL - It deserializes the in-memory IR given by the NPU driver and prepares it for the compiler
|
||||
|
||||
The interface between plugin and driver is based on an in-memory IR to facilitate backward and forward compatibility between two software packages (OpenVINO and NPU driver) that inherently have a different release cadence.
|
||||
|
||||
|
||||
## Model Execution
|
||||
|
||||
NPU plugin will use the Level Zero (L0) API to execute the precompiled model on the NPU Device. The inference is executed as a standard L0 workload by describing the required tasks inside a command list and by submitting the list to the command queue for execution. The plugin will not use the CPU to execute any part of the inference workload. No pre/post processing workloads are executed on the CPU either, the entire inference will be offloaded on the NPU device.
|
||||
|
||||
### Device management
|
||||
|
||||
There is currently no support for multiple devices (N x discrete + integrated), one single level zero device will be enumerated during level zero backend initialization. Support for multiple devices will be added in future releases.
|
||||
|
||||
### Inference pipeline
|
||||
|
||||
The result of the model compilation is represented through a NetworkDescription. This model description is passed by the plugin to the driver to create a level zero graph instance and obtain a graph handle that can later be used to execute multiple inferences in parallel for the same model. Since the same model instance is shared across all subsequent inference objects, this initialization step is performed by default right after the model is compiled and it can be postponed until the creation of the first inference request through the use of an environment variable: "IE_NPU_CREATE_EXECUTOR" (IE_NPU_CREATE_EXECUTOR=0 to postpone the initialization).
|
||||
|
||||
Users can create one or more inference requests for a compiled model using OpenVINO API:
|
||||
|
||||
```
|
||||
ov::InferRequest request = compiled_model.create_infer_request();
|
||||
```
|
||||
|
||||
One unique level zero command queue is currently used to execute all inference requests created for the same model.
|
||||
|
||||
|
||||
### Memory allocation
|
||||
|
||||
Each inference request is linked to an already compiled model and maintains a reference to the common model description. The level zero (user mode) driver parses the model description and returns the number and size of input/output buffers that need to be allocated per inference request. NPU plugin allocates input/output buffers through dedicated level zero API. Pointers to input/output buffers need to be provided when the command list is created so a different command list is used for each inference request.
|
||||
|
||||
The set of input and output buffers allocated by the NPU plugin for each inference request can be retrieved using the following API:
|
||||
```
|
||||
ov::Tensor requestTensor = inferRequest.get_tensor(tensor_name);
|
||||
```
|
||||
|
||||
Users can access the "data" member of the specified tensor and can directly populate it or read it. This is the recommended usage of input and output buffers that guarantees no extra copies will be performed by the NPU plugin.
|
||||
|
||||
Alternatively, users can configure different input or output buffers to be used during inference:
|
||||
```
|
||||
inferRequest.set_tensor(tensor_name, tensor);
|
||||
```
|
||||
|
||||
Since these tensors are not natively accessible by the NPU device, plugin will perform the required memory copies to and from the original buffers that were allocated when the inference request was created. This has an impact on the inference latency and should be avoided whenever possible.
|
||||
|
||||
Once the inference request is created and input/output tensors are prepared for the inference, the execution can be triggered either synchronously or asynchronously:
|
||||
* Synchronous execution:
|
||||
```
|
||||
inferRequest.infer();
|
||||
```
|
||||
* Asynchronous execution using:
|
||||
```
|
||||
inferRequest.start_async();
|
||||
inferRequest.wait(); // optional, in case user callback is not provided
|
||||
```
|
||||
|
||||
Multiple inferences can be executed in parallel, either from the same application thread through the use of asynchronous methods or from multiple threads with any of the available methods (synchronous or asynchronous). There is an optimal number of inference requests to be executed in parallel that would yield the best throughput without impacting the latency observed for each inference. This optimal number of requests is different for each model and depends on the ratio between the duration of the model execution on the DPU HW and the rest of the latency required to pass the request through the entire software stack. The NPU plugin returns the optimal number of inference requests through a dedicated property (`ov::optimal_number_of_infer_requests`).
|
||||
|
||||
Note: the current implementation of this property does not estimate or check the duration of the model execution. A fixed number of recommended inference requests is currently returned based on the existing performance data gathered from a reduced set of models with different topologies.
|
||||
|
||||
|
||||
## Supported Properties
|
||||
|
||||
Properties can be used to query and adjust the behavior of the NPU plugin itself or various parameters that control model compilation and execution.
|
||||
|
||||
The following methods are made available to return the value of a given property (at core level or model specific):
|
||||
```
|
||||
plugin_properties = ov.get_property("NPU", <property_name>);
|
||||
[...]
|
||||
model_properties = compiled_model.get_property(<property_name>);
|
||||
```
|
||||
|
||||
The following methods are made available to set the value of a given property (at core level or model specific):
|
||||
```
|
||||
ov.set_property("NPU", {{Key, Value}});
|
||||
[...]
|
||||
compiled_model.set_property({{Key, Value}});
|
||||
```
|
||||
|
||||
The following properties are supported:
|
||||
|
||||
| Parameter Name | | Description | Supported Values | Default Value |
|
||||
| :--- | :--- | :--- |:--- |:-- |
|
||||
| `ov::supported_properties`/</br>`SUPPORTED_METRICS`/</br>`SUPPORTED_CONFIG_KEYS` | RO | Returns a list of all supported properties.</br> Can be queried on runtime. | `N/A` | `N/A` |
|
||||
| `ov::caching_properties`/</br>`CACHING_PROPERTIES` | RW | Returns a list of all properties that are used by OpenVINO cache to build the hash key. | `N/A` | `N/A` |
|
||||
| `ov::compilation_num_threads`/</br>`COMPILATION_NUM_THREADS` | RW | Maximum number of threads that can be used for compilation tasks. | `N/A` | `N/A` |
|
||||
| `ov::num_streams`/</br>`NUM_STREAMS` | RO | Not used by the NPU plugin.</br> Always set to 1. | `AUTO/`</br>`INT` | `1` |
|
||||
| `ov::optimal_number_of_infer_requests`/</br>`OPTIMAL_NUMBER_OF_INFER_REQUESTS` | RO | Returns the optimal number of inference requests to be used by the application. Depends on the platform version and on ov::hint::performance_mode. Please see the table below. | `N/A` | `N/A` |
|
||||
| `ov::range_for_async_infer_requests`/</br>`RANGE_FOR_ASYNC_INFER_REQUESTS` | RO | Returns a tuple (bottom, top, step). </br> Not used by the NPU plugin. | `N/A` | `N/A` |
|
||||
| `ov::range_for_streams`/</br>`RANGE_FOR_STREAMS` | RO | Returns a tuple (bottom, top).</br> Not used by the NPU plugin. | `N/A`| `N/A` |
|
||||
| `ov::enable_profiling`/</br>`PERF_COUNT` | RW | Enables or disables performance counters. | `YES`/ `NO` | `NO` |
|
||||
| `ov::hint::performance_mode`/</br>`PERFORMANCE_HINT` | RW | Sets the performance profile used to determine default values of DPUs/DMAs/NIREQs.</br>Default values for each profile are documented below. | `THROUGHPUT`/</br>`LATENCY`/</br>`UNDEFINED` | `UNDEFINED` |
|
||||
| `ov::hint::num_requests`/</br>`PERFORMANCE_HINT_NUM_REQUESTS` | RW | Sets the number of outstanding inference requests. | `[0-]` | `1` |
|
||||
| `ov::hint::model_priority`/</br>`MODEL_PRIORITY` | RW | Assigns a priority for the model execution. | `LOW`/</br>`MEDIUM`/</br>`HIGH` | `MEDIUM` |
|
||||
| `ov::hint::enable_cpu_pinning`/</br>`ENABLE_CPU_PINNING` | RW | Allows CPU threads pinning during inference. | `YES`/ `NO` /</br>`NO`
|
||||
| `ov::log::level`/</br>`LOG_LEVEL` | RW | Sets the log level for NPU Plugin. An environment variable is also made available to expose logs from early initialization phase: OV_NPU_LOG_LEVEL. | `LOG_LEVEL_NONE`/</br>`LOG_LEVEL_ERROR`/</br>`LOG_LEVEL_WARNING`/</br>`LOG_LEVEL_DEBUG`/</br>`LOG_LEVEL_TRACE` | `_NONE` |
|
||||
| `ov::cache_dir`/</br>`CACHE_DIR` | RW | Folder path to be used by the OpenVINO cache. | `N/A` | empty |
|
||||
| `ov::available_devices`/</br>`AVAILABLE_DEVICES` | RO | Returns the list of enumerated NPU devices. </br> NPU plugin does not currently support multiple devices. | `N/A`| `N/A` |
|
||||
| `ov::device::id`/</br>`DEVICE_ID` | RW | Device identifier. Empty means auto detection. | empty/</br> `3700`/</br> `3720` | empty |
|
||||
| `ov::device::uuid`/</br> | RO | Returns the Universal Unique ID of the NPU device. | `N/A`| `N/A` |
|
||||
| `ov::device::architecture`/</br>`DEVICE_ARCHITECTURE` | RO | Returns the platform information. | `N/A`| `N/A` |
|
||||
| `ov::device::full_name`/</br>`FULL_DEVICE_NAME` | RO | Returns the full name of the NPU device. | `N/A`| `N/A` |
|
||||
| `ov::internal::exclusive_async_requests`/</br>`EXCLUSIVE_ASYNC_REQUESTS` | RW | Allows to use exclusive task executor for asynchronous infer requests. | `YES`/ `NO`| `NO` |
|
||||
|
||||
|
||||
### Performance Hint: Default Number of DPU Groups / DMA Engines
|
||||
|
||||
The following table shows the default values for the number of DPU Groups (Tiles) and DMA Engines selected by the plugin based on the performance mode (THROUGHPUT/LATENCY) and based on the platform:
|
||||
|
||||
| Performance hint | NPU Platform | Number of DPU Groups | Number of DMA Engines |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| THROUGHPUT | 3700 | 1 | 1 |
|
||||
| THROUGHPUT | 3720 | 2 (all of them) | 2 (all of them) |
|
||||
| LATENCY | 3700 | 4 (all of them) | 1 |
|
||||
| LATENCY | 3720 | 2 (all of them) | 2 (all of them) |
|
||||
|
||||
|
||||
### Performance Hint: Optimal Number of Inference Requests
|
||||
|
||||
The following table shows the optimal number of inference requests returned by the plugin based on the performance mode (THROUGHPUT/LATENCY) and based on the platform:
|
||||
|
||||
| NPU Platform | Nr. of Inference Requests </br> THROUGHPUT | Nr. of Inference Requests </br> LATENCY |
|
||||
| :--- | :--- | :--- |
|
||||
| 3700 | 8 | 1 |
|
||||
| 3720 | 4 | 1 |
|
||||
|
||||
|
||||
|
||||
## Stateful models
|
||||
|
||||
Key ingredients to support stateful models which distinguish them from other models are:
|
||||
* Implementing ReadValue and Assign operators
|
||||
* Implementing Query State API (to give user an API to reset/get/set states)
|
||||
* Implementing initialization for a state
|
||||
|
||||
More details on how OpenVINO supports stateful models can be found here: [Stateful models](https://docs.openvino.ai/2022.3/openvino_docs_OV_UG_network_state_intro.html).
|
||||
|
||||
The current implementation of state variables inside the NPU plugin is illustrated by the below diagram:
|
||||
|
||||
|
||||
|
||||
Notes on the implementation:
|
||||
* One network with N inputs + K state variables + M outputs will be converted by the compiler into a model with (N+K) inputs and (M+K) outputs. State variables are represented by a set of input/output nodes. This is currently needed because the underlying software stack (driver and runtime) does not support state variables.
|
||||
* The input and output nodes corresponding to state variables have different buffers allocated through the Level Zero API.
|
||||
* The content of the output buffer is copied back into the input buffer by the plugin through the use of an intermediate state buffer:
|
||||
* NPU Plugin allocates and maintains one additional state buffer which is exposed through the GetState/SetState API
|
||||
* The actual level zero input buffer for the state is updated when the inference is triggered with the content of the state buffer
|
||||
* The state buffer is updated once the inference is completed with the content of the output level zero buffer
|
||||
|
||||
The implementation of state variables in the NPU plugin will be improved for upcoming releases.
|
||||
|
||||
|
||||
## Dynamic shapes
|
||||
Dynamic shapes are not supported by the NPU plugin yet.
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
ov_option(ENABLE_MLIR_COMPILER "Enable compilation of npu_mlir_compiler libraries" ON)
|
||||
|
||||
ov_option(BUILD_COMPILER_FOR_DRIVER "Enable build of npu_driver_compiler" OFF)
|
||||
|
||||
ov_dependent_option(ENABLE_DRIVER_COMPILER_ADAPTER "Enable NPU Compiler inside driver" ON "NOT BUILD_COMPILER_FOR_DRIVER" OFF)
|
||||
|
||||
if(NOT BUILD_SHARED_LIBS AND NOT ENABLE_MLIR_COMPILER AND NOT ENABLE_DRIVER_COMPILER_ADAPTER)
|
||||
message(FATAL_ERROR "No compiler found for static build!")
|
||||
endif()
|
||||
|
||||
# if ENABLE_ZEROAPI_BACKEND=ON, it adds the ze_loader dependency for driver compiler
|
||||
ov_dependent_option(ENABLE_ZEROAPI_BACKEND "Enable zero-api as a plugin backend" ON "NOT BUILD_COMPILER_FOR_DRIVER" OFF)
|
||||
|
||||
ov_dependent_option(ENABLE_IMD_BACKEND "Enable InferenceManagerDemo based NPU AL backend" OFF "NOT WIN32;NOT CMAKE_CROSSCOMPILING" OFF)
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5de0598bb4f332011f2e7ed1dfc2c8ff535eb572c586bc1e1a9df63101c8499b
|
||||
size 20299
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6e3b9dade2b9e6a29919174f2ab2c03f370c5832ae191657d6822a8966bbd55d
|
||||
size 49296
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
# Close LTO temporarily for add_symbol_to_partition_1 issue
|
||||
set(ENABLE_LTO OFF)
|
||||
|
||||
add_subdirectory(utils)
|
||||
|
||||
add_subdirectory(al)
|
||||
|
||||
if(ENABLE_DRIVER_COMPILER_ADAPTER)
|
||||
add_subdirectory(compiler)
|
||||
endif()
|
||||
|
||||
if(ENABLE_ZEROAPI_BACKEND)
|
||||
add_subdirectory(backend)
|
||||
endif()
|
||||
|
||||
if (NOT BUILD_COMPILER_FOR_DRIVER)
|
||||
add_subdirectory(plugin)
|
||||
endif()
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set(TARGET_NAME openvino_npu_al)
|
||||
|
||||
file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h)
|
||||
source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES})
|
||||
|
||||
add_library(${TARGET_NAME} STATIC ${SOURCES})
|
||||
add_library(openvino::npu_al ALIAS ${TARGET_NAME})
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES EXPORT_NAME npu_al)
|
||||
|
||||
target_include_directories(${TARGET_NAME}
|
||||
PUBLIC
|
||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
|
||||
)
|
||||
|
||||
if(ENABLE_MLIR_COMPILER)
|
||||
target_compile_definitions(${TARGET_NAME} PRIVATE ENABLE_MLIR_COMPILER)
|
||||
endif()
|
||||
|
||||
target_link_libraries(${TARGET_NAME}
|
||||
PUBLIC
|
||||
openvino::npu_logger_utils
|
||||
openvino::runtime::dev
|
||||
)
|
||||
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
|
||||
|
||||
#
|
||||
# targets install
|
||||
#
|
||||
ov_install_static_lib(${TARGET_NAME} ${NPU_PLUGIN_COMPONENT})
|
||||
|
||||
ov_developer_package_export_targets(TARGET openvino::npu_al
|
||||
INSTALL_INCLUDE_DIRECTORIES
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/include)
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
|
||||
#include "npu_private_properties.hpp"
|
||||
|
||||
namespace utils {
|
||||
bool isNPUDevice(const uint32_t deviceId);
|
||||
uint32_t getSliceIdBySwDeviceId(const uint32_t swDevId);
|
||||
std::string getPlatformByDeviceName(const std::string& deviceName);
|
||||
} // namespace utils
|
||||
|
|
@ -0,0 +1,249 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "intel_npu/al/config/config.hpp"
|
||||
#include "npu_private_properties.hpp"
|
||||
#include "openvino/runtime/internal_properties.hpp"
|
||||
#include "openvino/runtime/properties.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
//
|
||||
// register
|
||||
//
|
||||
|
||||
void registerCommonOptions(OptionsDesc& desc);
|
||||
|
||||
//
|
||||
// PERFORMANCE_HINT
|
||||
//
|
||||
|
||||
struct PERFORMANCE_HINT final : OptionBase<PERFORMANCE_HINT, ov::hint::PerformanceMode> {
|
||||
static std::string_view key() {
|
||||
return ov::hint::performance_mode.name();
|
||||
}
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
return "ov::hint::PerformanceMode";
|
||||
}
|
||||
|
||||
static ov::hint::PerformanceMode defaultValue() {
|
||||
return ov::hint::PerformanceMode::LATENCY;
|
||||
}
|
||||
|
||||
static ov::hint::PerformanceMode parse(std::string_view val);
|
||||
};
|
||||
|
||||
// PERFORMANCE_HINT_NUM_REQUESTS
|
||||
//
|
||||
|
||||
struct PERFORMANCE_HINT_NUM_REQUESTS final : OptionBase<PERFORMANCE_HINT_NUM_REQUESTS, uint32_t> {
|
||||
static std::string_view key() {
|
||||
return ov::hint::num_requests.name();
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Returns configuration value if it is valid, otherwise throws
|
||||
* @details This is the same function as "InferenceEngine::PerfHintsConfig::CheckPerformanceHintRequestValue",
|
||||
* slightly modified as to not rely on the legacy API anymore.
|
||||
* @param configuration value as string
|
||||
* @return configuration value as number
|
||||
*/
|
||||
static uint32_t parse(std::string_view val) {
|
||||
int val_i = -1;
|
||||
try {
|
||||
val_i = std::stoi(val.data());
|
||||
if (val_i >= 0)
|
||||
return val_i;
|
||||
else
|
||||
throw std::logic_error("wrong val");
|
||||
} catch (const std::exception&) {
|
||||
OPENVINO_THROW("Wrong value of ",
|
||||
val.data(),
|
||||
" for property key ",
|
||||
ov::hint::num_requests.name(),
|
||||
". Expected only positive integer numbers");
|
||||
}
|
||||
}
|
||||
|
||||
static uint32_t defaultValue() {
|
||||
// Default value depends on PERFORMANCE_HINT, see getOptimalNumberOfInferRequestsInParallel
|
||||
// 1 corresponds to LATENCY and default mode (hints not specified)
|
||||
return 1u;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// INFERENCE_PRECISION_HINT
|
||||
//
|
||||
|
||||
struct INFERENCE_PRECISION_HINT final : OptionBase<INFERENCE_PRECISION_HINT, ov::element::Type> {
|
||||
static std::string_view key() {
|
||||
return ov::hint::inference_precision.name();
|
||||
}
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
return "ov::hint::inference_precision";
|
||||
}
|
||||
|
||||
static ov::element::Type defaultValue() {
|
||||
return ov::element::f16;
|
||||
}
|
||||
|
||||
static ov::element::Type parse(std::string_view val) {
|
||||
if (val.empty() || (val == "f16")) {
|
||||
return ov::element::f16;
|
||||
} else if (val == "i8") {
|
||||
return ov::element::i8;
|
||||
} else {
|
||||
OPENVINO_THROW("Wrong value ",
|
||||
val.data(),
|
||||
" for property key ",
|
||||
ov::hint::inference_precision.name(),
|
||||
". Supported values: f16, i8");
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
//
|
||||
// PERF_COUNT
|
||||
//
|
||||
|
||||
struct PERF_COUNT final : OptionBase<PERF_COUNT, bool> {
|
||||
static std::string_view key() {
|
||||
return ov::enable_profiling.name();
|
||||
}
|
||||
|
||||
static bool defaultValue() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// LOG_LEVEL
|
||||
//
|
||||
|
||||
struct LOG_LEVEL final : OptionBase<LOG_LEVEL, ov::log::Level> {
|
||||
static std::string_view key() {
|
||||
return ov::log::level.name();
|
||||
}
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
return "ov::log::Level";
|
||||
}
|
||||
|
||||
static std::string_view envVar() {
|
||||
return "OV_NPU_LOG_LEVEL";
|
||||
}
|
||||
|
||||
static ov::log::Level defaultValue() {
|
||||
return ov::log::Level::NO;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// PLATFORM
|
||||
//
|
||||
|
||||
struct PLATFORM final : OptionBase<PLATFORM, std::string> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::platform.name();
|
||||
}
|
||||
|
||||
static std::string defaultValue() {
|
||||
return std::string(ov::intel_npu::Platform::AUTO_DETECT);
|
||||
}
|
||||
|
||||
#ifdef NPU_PLUGIN_DEVELOPER_BUILD
|
||||
static std::string_view envVar() {
|
||||
return "IE_NPU_PLATFORM";
|
||||
}
|
||||
#endif
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// DEVICE_ID
|
||||
//
|
||||
|
||||
struct DEVICE_ID final : OptionBase<DEVICE_ID, std::string> {
|
||||
static std::string_view key() {
|
||||
return ov::device::id.name();
|
||||
}
|
||||
|
||||
static std::string defaultValue() {
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// CACHE_DIR
|
||||
//
|
||||
|
||||
struct CACHE_DIR final : OptionBase<CACHE_DIR, std::string> {
|
||||
static std::string_view key() {
|
||||
return ov::cache_dir.name();
|
||||
}
|
||||
|
||||
static std::string defaultValue() {
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// LOADED_FROM_CACHE
|
||||
//
|
||||
|
||||
struct LOADED_FROM_CACHE final : OptionBase<LOADED_FROM_CACHE, bool> {
|
||||
static std::string_view key() {
|
||||
return ov::loaded_from_cache.name();
|
||||
}
|
||||
|
||||
static bool defaultValue() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// CACHING PROPERTIES
|
||||
//
|
||||
|
||||
struct CACHING_PROPERTIES final : OptionBase<CACHING_PROPERTIES, std::string> {
|
||||
static std::string_view key() {
|
||||
return ov::internal::caching_properties.name();
|
||||
}
|
||||
|
||||
static std::string defaultValue() {
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// INTERNAL SUPPORTED PROPERTIES
|
||||
//
|
||||
|
||||
struct INTERNAL_SUPPORTED_PROPERTIES final : OptionBase<INTERNAL_SUPPORTED_PROPERTIES, std::string> {
|
||||
static std::string_view key() {
|
||||
return ov::internal::supported_properties.name();
|
||||
}
|
||||
|
||||
static std::string defaultValue() {
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
||||
namespace ov {
|
||||
namespace hint {
|
||||
|
||||
std::string_view stringifyEnum(PerformanceMode val);
|
||||
|
||||
} // namespace hint
|
||||
} // namespace ov
|
||||
|
|
@ -0,0 +1,363 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <thread>
|
||||
|
||||
#include "common.hpp"
|
||||
#include "intel_npu/al/config/config.hpp"
|
||||
#include "npu_private_properties.hpp"
|
||||
#include "openvino/runtime/intel_npu/properties.hpp"
|
||||
|
||||
namespace ov {
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
std::string_view stringifyEnum(CompilerType val);
|
||||
std::string_view stringifyEnum(ElfCompilerBackend val);
|
||||
|
||||
} // namespace intel_npu
|
||||
|
||||
} // namespace ov
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
//
|
||||
// register
|
||||
//
|
||||
|
||||
void registerCompilerOptions(OptionsDesc& desc);
|
||||
|
||||
//
|
||||
// COMPILER_TYPE
|
||||
//
|
||||
|
||||
struct COMPILER_TYPE final : OptionBase<COMPILER_TYPE, ov::intel_npu::CompilerType> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::compiler_type.name();
|
||||
}
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
return "ov::intel_npu::CompilerType";
|
||||
}
|
||||
|
||||
static std::string_view envVar();
|
||||
|
||||
static ov::intel_npu::CompilerType defaultValue();
|
||||
|
||||
static ov::intel_npu::CompilerType parse(std::string_view val);
|
||||
|
||||
static std::string toString(const ov::intel_npu::CompilerType& val);
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// COMPILATION_MODE
|
||||
//
|
||||
|
||||
struct COMPILATION_MODE final : OptionBase<COMPILATION_MODE, std::string> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::compilation_mode.name();
|
||||
}
|
||||
|
||||
#ifdef NPU_PLUGIN_DEVELOPER_BUILD
|
||||
static std::string_view envVar() {
|
||||
return "IE_NPU_COMPILATION_MODE";
|
||||
}
|
||||
#endif
|
||||
|
||||
static std::string defaultValue() {
|
||||
return "";
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// DYNAMIC_SHAPE_TO_STATIC
|
||||
//
|
||||
|
||||
struct DYNAMIC_SHAPE_TO_STATIC final : OptionBase<DYNAMIC_SHAPE_TO_STATIC, bool> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::dynamic_shape_to_static.name();
|
||||
}
|
||||
|
||||
#ifdef NPU_PLUGIN_DEVELOPER_BUILD
|
||||
static std::string_view envVar() {
|
||||
return "IE_NPU_DYNAMIC_SHAPE_TO_STATIC";
|
||||
}
|
||||
#endif
|
||||
|
||||
static bool defaultValue() {
|
||||
return false;
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// COMPILATION_MODE_PARAMS
|
||||
//
|
||||
|
||||
struct COMPILATION_MODE_PARAMS final : OptionBase<COMPILATION_MODE_PARAMS, std::string> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::compilation_mode_params.name();
|
||||
}
|
||||
|
||||
static std::string defaultValue() {
|
||||
return {};
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// DPU_GROUPS
|
||||
//
|
||||
|
||||
struct DPU_GROUPS final : OptionBase<DPU_GROUPS, int64_t> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::dpu_groups.name();
|
||||
}
|
||||
|
||||
static std::vector<std::string_view> deprecatedKeys() {
|
||||
return {};
|
||||
}
|
||||
|
||||
static int64_t defaultValue() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifdef NPU_PLUGIN_DEVELOPER_BUILD
|
||||
static std::string_view envVar() {
|
||||
return "IE_NPU_DPU_GROUPS";
|
||||
}
|
||||
#endif
|
||||
};
|
||||
|
||||
//
|
||||
// SELECTED_TILES
|
||||
//
|
||||
|
||||
struct TILES final : OptionBase<TILES, int64_t> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::tiles.name();
|
||||
}
|
||||
|
||||
static std::vector<std::string_view> deprecatedKeys() {
|
||||
return {};
|
||||
}
|
||||
|
||||
static int64_t defaultValue() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifdef NPU_PLUGIN_DEVELOPER_BUILD
|
||||
static std::string_view envVar() {
|
||||
return "IE_NPU_TILES";
|
||||
}
|
||||
#endif
|
||||
};
|
||||
|
||||
//
|
||||
// STEPPING
|
||||
//
|
||||
|
||||
struct STEPPING final : OptionBase<STEPPING, int64_t> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::stepping.name();
|
||||
}
|
||||
|
||||
static std::vector<std::string_view> deprecatedKeys() {
|
||||
return {};
|
||||
}
|
||||
|
||||
static int64_t defaultValue() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// MAX_TILES
|
||||
//
|
||||
|
||||
struct MAX_TILES final : OptionBase<MAX_TILES, int64_t> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::max_tiles.name();
|
||||
}
|
||||
|
||||
static std::vector<std::string_view> deprecatedKeys() {
|
||||
return {};
|
||||
}
|
||||
|
||||
static int64_t defaultValue() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// DMA_ENGINES
|
||||
//
|
||||
|
||||
struct DMA_ENGINES final : OptionBase<DMA_ENGINES, int64_t> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::dma_engines.name();
|
||||
}
|
||||
|
||||
static std::vector<std::string_view> deprecatedKeys() {
|
||||
return {};
|
||||
}
|
||||
|
||||
static int64_t defaultValue() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifdef NPU_PLUGIN_DEVELOPER_BUILD
|
||||
static std::string_view envVar() {
|
||||
return "IE_NPU_DMA_ENGINES";
|
||||
}
|
||||
#endif
|
||||
};
|
||||
|
||||
//
|
||||
// USE_ELF_COMPILER_BACKEND
|
||||
//
|
||||
|
||||
struct USE_ELF_COMPILER_BACKEND final : OptionBase<USE_ELF_COMPILER_BACKEND, ov::intel_npu::ElfCompilerBackend> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::use_elf_compiler_backend.name();
|
||||
}
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
return "ov::intel_npu::ElfCompilerBackend";
|
||||
}
|
||||
|
||||
#ifdef NPU_PLUGIN_DEVELOPER_BUILD
|
||||
static std::string_view envVar() {
|
||||
return "IE_NPU_USE_ELF_COMPILER_BACKEND";
|
||||
}
|
||||
#endif
|
||||
|
||||
static ov::intel_npu::ElfCompilerBackend defaultValue() {
|
||||
return ov::intel_npu::ElfCompilerBackend::AUTO;
|
||||
}
|
||||
|
||||
static ov::intel_npu::ElfCompilerBackend parse(std::string_view val);
|
||||
|
||||
static std::string toString(const ov::intel_npu::ElfCompilerBackend& val);
|
||||
};
|
||||
|
||||
//
|
||||
// BACKEND_COMPILATION_PARAMS
|
||||
//
|
||||
|
||||
struct BACKEND_COMPILATION_PARAMS final : OptionBase<BACKEND_COMPILATION_PARAMS, std::string> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::backend_compilation_params.name();
|
||||
}
|
||||
|
||||
static std::string defaultValue() {
|
||||
return {};
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// COMPILATION_NUM_THREADS
|
||||
//
|
||||
|
||||
struct COMPILATION_NUM_THREADS final : OptionBase<COMPILATION_NUM_THREADS, int32_t> {
|
||||
static std::string_view key() {
|
||||
return ov::compilation_num_threads.name();
|
||||
}
|
||||
|
||||
static int32_t defaultValue() {
|
||||
return std::max(1, static_cast<int>(std::thread::hardware_concurrency()));
|
||||
}
|
||||
|
||||
static void validateValue(const int32_t& num) {
|
||||
if (num <= 0) {
|
||||
OPENVINO_THROW("ov::compilation_num_threads must be positive int32 value");
|
||||
}
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::CompileTime;
|
||||
}
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,444 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cassert>
|
||||
#include <chrono>
|
||||
#include <functional>
|
||||
#include <iomanip>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "openvino/core/except.hpp"
|
||||
#include "openvino/runtime/properties.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
template <class T>
|
||||
struct TypePrinter {
|
||||
static constexpr bool hasName() {
|
||||
return false;
|
||||
}
|
||||
static constexpr const char* name();
|
||||
};
|
||||
|
||||
#define TYPE_PRINTER(type) \
|
||||
template <> \
|
||||
struct TypePrinter<type> { \
|
||||
static constexpr bool hasName() { \
|
||||
return true; \
|
||||
} \
|
||||
static constexpr const char* name() { \
|
||||
return #type; \
|
||||
} \
|
||||
};
|
||||
|
||||
TYPE_PRINTER(bool)
|
||||
TYPE_PRINTER(char)
|
||||
TYPE_PRINTER(char*)
|
||||
TYPE_PRINTER(int)
|
||||
TYPE_PRINTER(unsigned int)
|
||||
TYPE_PRINTER(int64_t)
|
||||
TYPE_PRINTER(double)
|
||||
TYPE_PRINTER(std::string)
|
||||
|
||||
//
|
||||
// OptionParser
|
||||
//
|
||||
|
||||
template <typename T>
|
||||
struct OptionParser;
|
||||
|
||||
template <>
|
||||
struct OptionParser<std::string> final {
|
||||
static std::string parse(std::string_view val) {
|
||||
return {val.data(), val.size()};
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OptionParser<bool> final {
|
||||
static bool parse(std::string_view val);
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OptionParser<int32_t> final {
|
||||
static int32_t parse(std::string_view val);
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OptionParser<int64_t> final {
|
||||
static int64_t parse(std::string_view val);
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OptionParser<uint64_t> final {
|
||||
static uint64_t parse(std::string_view val);
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OptionParser<double> final {
|
||||
static double parse(std::string_view val);
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OptionParser<ov::log::Level> final {
|
||||
static ov::log::Level parse(std::string_view val);
|
||||
};
|
||||
|
||||
void splitAndApply(const std::string& str, char delim, std::function<void(std::string_view)> callback);
|
||||
|
||||
template <typename T>
|
||||
struct OptionParser<std::vector<T>> final {
|
||||
static std::vector<T> parse(std::string_view val) {
|
||||
std::vector<T> res;
|
||||
splitAndApply(val, ',', [&](std::string_view item) {
|
||||
res.push_back(OptionParser<T>::parse(item));
|
||||
});
|
||||
return res;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Rep, typename Period>
|
||||
struct OptionParser<std::chrono::duration<Rep, Period>> final {
|
||||
static std::chrono::duration<Rep, Period> parse(std::string_view val) {
|
||||
std::istringstream stream(val.data());
|
||||
|
||||
Rep count{};
|
||||
if (stream >> count) {
|
||||
OPENVINO_ASSERT(count >= 0,
|
||||
"Value '",
|
||||
count,
|
||||
"' is not a valid time duration, non-negative values expected");
|
||||
return std::chrono::duration<Rep, Period>(count);
|
||||
}
|
||||
|
||||
OPENVINO_THROW("Can't parse '", val.data(), "' as time duration");
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// OptionPrinter
|
||||
//
|
||||
|
||||
template <typename T>
|
||||
struct OptionPrinter final {
|
||||
static std::string toString(const T& val) {
|
||||
std::stringstream ss;
|
||||
if constexpr (std::is_floating_point_v<std::decay_t<T>>) {
|
||||
ss << std::fixed << std::setprecision(2) << val;
|
||||
} else if constexpr (std::is_enum_v<std::decay_t<T>>) {
|
||||
ss << stringifyEnum(val);
|
||||
return ss.str();
|
||||
} else {
|
||||
ss << val;
|
||||
}
|
||||
return ss.str();
|
||||
}
|
||||
};
|
||||
|
||||
// NB: boolean config option has values YES for true, NO for false
|
||||
template <>
|
||||
struct OptionPrinter<bool> final {
|
||||
static std::string toString(bool val);
|
||||
};
|
||||
|
||||
template <typename Rep, typename Period>
|
||||
struct OptionPrinter<std::chrono::duration<Rep, Period>> final {
|
||||
static std::string toString(const std::chrono::duration<Rep, Period>& val) {
|
||||
return std::to_string(val.count());
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OptionPrinter<ov::log::Level> final {
|
||||
static std::string toString(ov::log::Level val);
|
||||
};
|
||||
|
||||
//
|
||||
// OptionMode
|
||||
//
|
||||
|
||||
enum class OptionMode {
|
||||
Both,
|
||||
CompileTime,
|
||||
RunTime,
|
||||
};
|
||||
|
||||
std::string_view stringifyEnum(OptionMode val);
|
||||
|
||||
//
|
||||
// OptionBase
|
||||
//
|
||||
|
||||
// Actual Option description must inherit this class and pass itself as template parameter.
|
||||
template <class ActualOpt, typename T>
|
||||
struct OptionBase {
|
||||
using ValueType = T;
|
||||
|
||||
// `ActualOpt` must implement the following method:
|
||||
// static std::string_view key()
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
if constexpr (TypePrinter<T>::hasName()) {
|
||||
return TypePrinter<T>::name();
|
||||
}
|
||||
static_assert(TypePrinter<T>::hasName(),
|
||||
"Options type is not a standard type, please add `getTypeName()` to your option");
|
||||
}
|
||||
// Overload this to provide environment variable support.
|
||||
static std::string_view envVar() {
|
||||
return "";
|
||||
}
|
||||
|
||||
// Overload this to provide deprecated keys names.
|
||||
static std::vector<std::string_view> deprecatedKeys() {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Overload this to provide default value if it wasn't specified by user.
|
||||
// If it is std::nullopt - exception will be thrown in case of missing option access.
|
||||
static std::optional<T> defaultValue() {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// Overload this to provide more specific parser.
|
||||
static ValueType parse(std::string_view val) {
|
||||
return OptionParser<ValueType>::parse(val);
|
||||
}
|
||||
|
||||
// Overload this to provide more specific validation
|
||||
static void validateValue(const ValueType&) {}
|
||||
|
||||
// Overload this to provide more specific implementation.
|
||||
static OptionMode mode() {
|
||||
return OptionMode::Both;
|
||||
}
|
||||
|
||||
// Overload this for private options.
|
||||
static bool isPublic() {
|
||||
return true;
|
||||
}
|
||||
|
||||
static std::string toString(const ValueType& val) {
|
||||
return OptionPrinter<ValueType>::toString(val);
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// OptionValue
|
||||
//
|
||||
|
||||
namespace details {
|
||||
|
||||
class OptionValue {
|
||||
public:
|
||||
virtual ~OptionValue();
|
||||
|
||||
virtual std::string_view getTypeName() const = 0;
|
||||
virtual std::string toString() const = 0;
|
||||
};
|
||||
|
||||
template <typename Opt, typename T>
|
||||
class OptionValueImpl final : public OptionValue {
|
||||
using ToStringFunc = std::string (*)(const T&);
|
||||
|
||||
public:
|
||||
template <typename U>
|
||||
OptionValueImpl(U&& val, ToStringFunc toStringImpl) : _val(std::forward<U>(val)),
|
||||
_toStringImpl(toStringImpl) {}
|
||||
|
||||
std::string_view getTypeName() const override final {
|
||||
if constexpr (TypePrinter<T>::hasName()) {
|
||||
return TypePrinter<T>::name();
|
||||
} else {
|
||||
return Opt::getTypeName();
|
||||
}
|
||||
}
|
||||
|
||||
const T& getValue() const {
|
||||
return _val;
|
||||
}
|
||||
|
||||
std::string toString() const override {
|
||||
return _toStringImpl(_val);
|
||||
}
|
||||
|
||||
private:
|
||||
T _val;
|
||||
ToStringFunc _toStringImpl = nullptr;
|
||||
};
|
||||
|
||||
} // namespace details
|
||||
|
||||
//
|
||||
// OptionConcept
|
||||
//
|
||||
|
||||
namespace details {
|
||||
|
||||
struct OptionConcept final {
|
||||
std::string_view (*key)() = nullptr;
|
||||
std::string_view (*envVar)() = nullptr;
|
||||
OptionMode (*mode)() = nullptr;
|
||||
bool (*isPublic)() = nullptr;
|
||||
std::shared_ptr<OptionValue> (*validateAndParse)(std::string_view val) = nullptr;
|
||||
};
|
||||
|
||||
template <class Opt>
|
||||
std::shared_ptr<OptionValue> validateAndParse(std::string_view val) {
|
||||
using ValueType = typename Opt::ValueType;
|
||||
|
||||
try {
|
||||
auto parsedVal = Opt::parse(val);
|
||||
Opt::validateValue(parsedVal);
|
||||
return std::make_shared<OptionValueImpl<Opt, ValueType>>(std::move(parsedVal), &Opt::toString);
|
||||
} catch (const std::exception& e) {
|
||||
OPENVINO_THROW("Failed to parse '", Opt::key().data(), "' option : ", e.what());
|
||||
}
|
||||
}
|
||||
|
||||
template <class Opt>
|
||||
OptionConcept makeOptionModel() {
|
||||
return {&Opt::key, &Opt::envVar, &Opt::mode, &Opt::isPublic, &validateAndParse<Opt>};
|
||||
}
|
||||
|
||||
} // namespace details
|
||||
|
||||
//
|
||||
// OptionsDesc
|
||||
//
|
||||
|
||||
class OptionsDesc final {
|
||||
public:
|
||||
template <class Opt>
|
||||
void add();
|
||||
|
||||
std::vector<std::string> getSupported(bool includePrivate = false) const;
|
||||
|
||||
details::OptionConcept get(std::string_view key, OptionMode mode) const;
|
||||
void walk(std::function<void(const details::OptionConcept&)> cb) const;
|
||||
|
||||
private:
|
||||
std::unordered_map<std::string, details::OptionConcept> _impl;
|
||||
std::unordered_map<std::string, std::string> _deprecated;
|
||||
};
|
||||
|
||||
template <class Opt>
|
||||
void OptionsDesc::add() {
|
||||
OPENVINO_ASSERT(_impl.count(Opt::key().data()) == 0, "Option '", Opt::key().data(), "' was already registered");
|
||||
_impl.insert({Opt::key().data(), details::makeOptionModel<Opt>()});
|
||||
|
||||
for (const auto& deprecatedKey : Opt::deprecatedKeys()) {
|
||||
OPENVINO_ASSERT(_deprecated.count(deprecatedKey.data()) == 0,
|
||||
"Option '",
|
||||
deprecatedKey.data(),
|
||||
"' was already registered");
|
||||
_deprecated.insert({deprecatedKey.data(), Opt::key().data()});
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Config
|
||||
//
|
||||
|
||||
class Config final {
|
||||
public:
|
||||
using ConfigMap = std::map<std::string, std::string>;
|
||||
using ImplMap = std::unordered_map<std::string, std::shared_ptr<details::OptionValue>>;
|
||||
|
||||
explicit Config(const std::shared_ptr<const OptionsDesc>& desc);
|
||||
|
||||
void update(const ConfigMap& options, OptionMode mode = OptionMode::Both);
|
||||
|
||||
void parseEnvVars();
|
||||
|
||||
template <class Opt>
|
||||
bool has() const;
|
||||
|
||||
template <class Opt>
|
||||
typename Opt::ValueType get() const;
|
||||
|
||||
template <class Opt>
|
||||
typename std::string getString() const;
|
||||
|
||||
std::string toString() const;
|
||||
|
||||
private:
|
||||
std::shared_ptr<const OptionsDesc> _desc;
|
||||
ImplMap _impl;
|
||||
};
|
||||
|
||||
template <class Opt>
|
||||
bool Config::has() const {
|
||||
return _impl.count(Opt::key().data()) != 0;
|
||||
}
|
||||
|
||||
template <class Opt>
|
||||
typename Opt::ValueType Config::get() const {
|
||||
using ValueType = typename Opt::ValueType;
|
||||
|
||||
auto log = Logger::global().clone("Config");
|
||||
log.trace("Get value for the option '%s'", Opt::key().data());
|
||||
|
||||
const auto it = _impl.find(Opt::key().data());
|
||||
|
||||
if (it == _impl.end()) {
|
||||
const std::optional<ValueType> optional = Opt::defaultValue();
|
||||
log.trace("The option '%s' was not set by user, try default value", Opt::key().data());
|
||||
|
||||
OPENVINO_ASSERT(optional.has_value(),
|
||||
"Option '",
|
||||
Opt::key().data(),
|
||||
"' was not provided, no default value is available");
|
||||
return optional.value();
|
||||
}
|
||||
|
||||
OPENVINO_ASSERT(it->second != nullptr, "Got NULL OptionValue for :", Opt::key().data());
|
||||
|
||||
const auto optVal = std::dynamic_pointer_cast<details::OptionValueImpl<Opt, ValueType>>(it->second);
|
||||
#if defined(__CHROMIUMOS__)
|
||||
if (optVal == nullptr) {
|
||||
if (Opt::getTypeName() == it->second->getTypeName()) {
|
||||
const auto val = std::static_pointer_cast<details::OptionValueImpl<Opt, ValueType>>(it->second);
|
||||
return val->getValue();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
OPENVINO_ASSERT(optVal != nullptr,
|
||||
"Option '",
|
||||
Opt::key().data(),
|
||||
"' has wrong parsed type: expected '",
|
||||
Opt::getTypeName().data(),
|
||||
"', got '",
|
||||
it->second->getTypeName().data(),
|
||||
"'");
|
||||
|
||||
return optVal->getValue();
|
||||
}
|
||||
|
||||
template <class Opt>
|
||||
typename std::string Config::getString() const {
|
||||
typename Opt::ValueType value = Config::get<Opt>();
|
||||
|
||||
return Opt::toString(value);
|
||||
}
|
||||
|
||||
//
|
||||
// envVarStrToBool
|
||||
//
|
||||
|
||||
bool envVarStrToBool(const char* varName, const char* varValue);
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,185 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "intel_npu/al/config/config.hpp"
|
||||
#include "npu_private_properties.hpp"
|
||||
#include "openvino/runtime/intel_npu/properties.hpp"
|
||||
#include "openvino/runtime/internal_properties.hpp"
|
||||
#include "openvino/runtime/properties.hpp"
|
||||
|
||||
namespace ov {
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
std::string_view stringifyEnum(ProfilingType val);
|
||||
|
||||
} // namespace intel_npu
|
||||
|
||||
} // namespace ov
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
//
|
||||
// register
|
||||
//
|
||||
|
||||
void registerRunTimeOptions(OptionsDesc& desc);
|
||||
|
||||
//
|
||||
// EXCLUSIVE_ASYNC_REQUESTS
|
||||
//
|
||||
|
||||
struct EXCLUSIVE_ASYNC_REQUESTS final : OptionBase<EXCLUSIVE_ASYNC_REQUESTS, bool> {
|
||||
static std::string_view key() {
|
||||
return ov::internal::exclusive_async_requests.name();
|
||||
}
|
||||
|
||||
static bool defaultValue() {
|
||||
return false;
|
||||
}
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
return "bool";
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::RunTime;
|
||||
}
|
||||
};
|
||||
|
||||
int64_t getOptimalNumberOfInferRequestsInParallel(const Config& config);
|
||||
|
||||
///
|
||||
/// PROFILING TYPE
|
||||
///
|
||||
struct PROFILING_TYPE final : OptionBase<PROFILING_TYPE, ov::intel_npu::ProfilingType> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::profiling_type.name();
|
||||
}
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
return "ov::intel_npu::ProfilingType";
|
||||
}
|
||||
|
||||
static ov::intel_npu::ProfilingType defaultValue() {
|
||||
return ov::intel_npu::ProfilingType::MODEL;
|
||||
}
|
||||
|
||||
static ov::intel_npu::ProfilingType parse(std::string_view val);
|
||||
|
||||
static std::string toString(const ov::intel_npu::ProfilingType& val);
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::RunTime;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// MODEL_PRIORITY
|
||||
//
|
||||
|
||||
struct MODEL_PRIORITY final : OptionBase<MODEL_PRIORITY, ov::hint::Priority> {
|
||||
static std::string_view key() {
|
||||
return ov::hint::model_priority.name();
|
||||
}
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
return "ov::hint::Priority";
|
||||
}
|
||||
|
||||
static ov::hint::Priority defaultValue() {
|
||||
return ov::hint::Priority::MEDIUM;
|
||||
}
|
||||
|
||||
static ov::hint::Priority parse(std::string_view val);
|
||||
|
||||
static std::string toString(const ov::hint::Priority& val);
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::RunTime;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// CREATE_EXECUTOR
|
||||
//
|
||||
|
||||
struct CREATE_EXECUTOR final : OptionBase<CREATE_EXECUTOR, int64_t> {
|
||||
static std::string_view key() {
|
||||
return ov::intel_npu::create_executor.name();
|
||||
}
|
||||
|
||||
static int64_t defaultValue() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifdef NPU_PLUGIN_DEVELOPER_BUILD
|
||||
static std::string_view envVar() {
|
||||
return "IE_NPU_CREATE_EXECUTOR";
|
||||
}
|
||||
#endif
|
||||
|
||||
static bool isPublic() {
|
||||
return false;
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::RunTime;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// NUM_STREAMS
|
||||
//
|
||||
struct NUM_STREAMS final : OptionBase<NUM_STREAMS, ov::streams::Num> {
|
||||
static std::string_view key() {
|
||||
return ov::num_streams.name();
|
||||
}
|
||||
|
||||
static constexpr std::string_view getTypeName() {
|
||||
return "ov::streams::Num";
|
||||
}
|
||||
|
||||
const static ov::streams::Num defVal;
|
||||
|
||||
// The only supported number for currently supported platforms.
|
||||
// FIXME: update in the future
|
||||
static ov::streams::Num defaultValue() {
|
||||
return defVal;
|
||||
}
|
||||
|
||||
static ov::streams::Num parse(std::string_view val);
|
||||
|
||||
static std::string toString(const ov::streams::Num& val);
|
||||
|
||||
static void validateValue(const ov::streams::Num& num) {
|
||||
if (defVal != num && ov::streams::AUTO != num) {
|
||||
throw std::runtime_error("NUM_STREAMS can not be set");
|
||||
}
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::RunTime;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// ENABLE_CPU_PINNING
|
||||
//
|
||||
struct ENABLE_CPU_PINNING final : OptionBase<ENABLE_CPU_PINNING, bool> {
|
||||
static std::string_view key() {
|
||||
return ov::hint::enable_cpu_pinning.name();
|
||||
}
|
||||
|
||||
static bool defaultValue() {
|
||||
return false;
|
||||
}
|
||||
|
||||
static OptionMode mode() {
|
||||
return OptionMode::RunTime;
|
||||
}
|
||||
};
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
#include "intel_npu/al/icompiler.hpp"
|
||||
#include "openvino/runtime/icompiled_model.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
class ICompiledModel : public ov::ICompiledModel {
|
||||
public:
|
||||
using ov::ICompiledModel::ICompiledModel;
|
||||
|
||||
virtual const std::shared_ptr<const NetworkDescription>& get_network_description() const = 0;
|
||||
|
||||
virtual const Config& get_config() const = 0;
|
||||
|
||||
// Compiler is used for post-processing profiling data when using PERF_COUNT property
|
||||
virtual const ov::SoPtr<ICompiler>& get_compiler() const = 0;
|
||||
|
||||
const NetworkMetadata& get_network_metadata() const {
|
||||
return get_network_description()->metadata;
|
||||
}
|
||||
|
||||
protected:
|
||||
std::shared_ptr<const ICompiledModel> shared_from_this() const {
|
||||
return std::dynamic_pointer_cast<const ICompiledModel>(ov::ICompiledModel::shared_from_this());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
// Compiler Interface
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
|
||||
#include "intel_npu/al/config/config.hpp"
|
||||
#include "openvino/core/partial_shape.hpp"
|
||||
#include "openvino/core/type/element_type.hpp"
|
||||
#include "openvino/runtime/common.hpp"
|
||||
#include "openvino/runtime/profiling_info.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
/**
|
||||
* @brief A helper structure used for storing the metadata found within the I/O nodes.
|
||||
* @details The "legacyName" attribute holds the name most commonly used as map key for multiple structures.
|
||||
* This value also corresponds to the identifier used by the OpenVINO 1.0 API.
|
||||
*
|
||||
* "originalShape" corresponds to the shape registered in the graph, while "transposedShape" holds the shape obtained
|
||||
* upon applying a transposition corresponding to the legacy layout value. Use the "transposedShape" one if not sure
|
||||
* which one you need.
|
||||
*/
|
||||
struct IONodeDescriptor {
|
||||
std::string legacyName;
|
||||
std::string currentNodeName;
|
||||
std::unordered_set<std::string> outputTensorNames;
|
||||
ov::element::Type_t precision;
|
||||
ov::PartialShape originalShape;
|
||||
ov::PartialShape transposedShape;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief A helper map to represent descriptions for inputs and outputs
|
||||
* of a network
|
||||
*/
|
||||
using IONodeDescriptorMap = std::unordered_map<std::string, IONodeDescriptor>;
|
||||
|
||||
struct NetworkMetadata final {
|
||||
std::string name;
|
||||
|
||||
std::vector<std::string> inputNames;
|
||||
std::vector<std::string> outputNames;
|
||||
std::vector<std::string> stateNames;
|
||||
|
||||
IONodeDescriptorMap parameters;
|
||||
IONodeDescriptorMap results;
|
||||
IONodeDescriptorMap states;
|
||||
IONodeDescriptorMap profilingOutputs;
|
||||
|
||||
std::unordered_map<std::string, size_t> inputOrder;
|
||||
std::unordered_map<std::string, size_t> outputOrder;
|
||||
|
||||
int numStreams = 1;
|
||||
};
|
||||
|
||||
/**
|
||||
* @struct NetworkDescription
|
||||
* @brief The object returned by the compiler
|
||||
* to provide such information about a network as description of inputs and outputs,
|
||||
* name and compiled network in a format executable by device
|
||||
*/
|
||||
struct NetworkDescription final {
|
||||
NetworkDescription(std::vector<uint8_t>&& compiledNetwork, NetworkMetadata&& metadata)
|
||||
: compiledNetwork(std::move(compiledNetwork)),
|
||||
metadata(std::move(metadata)) {}
|
||||
// Force move semantics to prevent blob copies
|
||||
NetworkDescription(const NetworkDescription&) = delete;
|
||||
NetworkDescription(NetworkDescription&&) = default;
|
||||
NetworkDescription& operator=(const NetworkDescription&) = delete;
|
||||
NetworkDescription& operator=(NetworkDescription&&) = default;
|
||||
~NetworkDescription() = default;
|
||||
|
||||
std::vector<uint8_t> compiledNetwork;
|
||||
|
||||
NetworkMetadata metadata;
|
||||
};
|
||||
|
||||
/**
|
||||
* @interface ICompiler
|
||||
* @brief An interface to be implemented by a concrete compiler to provide
|
||||
* methods for preparing a network for execution on a NPU device
|
||||
*/
|
||||
class ICompiler : public std::enable_shared_from_this<ICompiler> {
|
||||
public:
|
||||
/**
|
||||
* @brief Returns the maximum OpenVino opset version supported by the compiler
|
||||
* @return opset version e.g. 11 for opset11
|
||||
*/
|
||||
virtual uint32_t getSupportedOpsetVersion() const = 0;
|
||||
|
||||
/**
|
||||
* @brief Transforms a network from the OpenVINO model representation to a format executable
|
||||
* by a NPU device
|
||||
* @param model a shared pointer to the OpenVINO model to be compiled
|
||||
* @param config a reference to NPUConfig containing plugin config options
|
||||
* including config options related to compilation
|
||||
* @return a shared pointer on an object implementing NetworkDescription interface
|
||||
*/
|
||||
virtual NetworkDescription compile(const std::shared_ptr<const ov::Model>& model, const Config& config) const = 0;
|
||||
|
||||
/**
|
||||
* @brief Returns information about supported layers of the network passed
|
||||
* @param model The model to be queried
|
||||
* @param config A reference to NPUConfig containing plugin config options
|
||||
* including config options related to compilation
|
||||
* @returns SupportedOpsMap structure with information about supported layers
|
||||
*/
|
||||
virtual ov::SupportedOpsMap query(const std::shared_ptr<const ov::Model>& model, const Config& config) const = 0;
|
||||
|
||||
/**
|
||||
* @brief Parses already compiled network to extract meta information:
|
||||
* inputs and outputs descriptions
|
||||
* @param network compiled network represented as a vector of char
|
||||
* @param config a reference to NPUConfig containing plugin config options
|
||||
* Note: compilation options will be ignored,
|
||||
* since the network is already compiled
|
||||
* @param netName a reference to the string describing network name
|
||||
* to be used for creating network description
|
||||
* @return a shared pointer on an object implementing NetworkDescription interface
|
||||
*/
|
||||
virtual NetworkMetadata parse(const std::vector<uint8_t>& network, const Config& config) const = 0;
|
||||
|
||||
virtual std::vector<ov::ProfilingInfo> process_profiling_output(const std::vector<uint8_t>& profData,
|
||||
const std::vector<uint8_t>& network,
|
||||
const Config& config) const = 0;
|
||||
|
||||
protected:
|
||||
virtual ~ICompiler() = default;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "openvino/itt.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
namespace itt {
|
||||
namespace domains {
|
||||
|
||||
OV_ITT_DOMAIN(NPUPlugin);
|
||||
OV_ITT_DOMAIN(LevelZeroBackend);
|
||||
|
||||
} // namespace domains
|
||||
} // namespace itt
|
||||
} // namespace intel_npu
|
||||
|
||||
namespace itt = ::intel_npu::itt;
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
//
|
||||
// Prefix for ReadValue and Assign operations in compiler.
|
||||
//
|
||||
#define READVALUE_PREFIX std::string("vpux_ie_read_value_")
|
||||
#define ASSIGN_PREFIX std::string("vpux_ie_assign_")
|
||||
|
||||
inline bool isStateInputName(const std::string& name) {
|
||||
return !name.compare(0, READVALUE_PREFIX.length(), READVALUE_PREFIX);
|
||||
}
|
||||
inline bool isStateOutputName(const std::string& name) {
|
||||
return !name.compare(0, ASSIGN_PREFIX.length(), ASSIGN_PREFIX);
|
||||
}
|
||||
|
||||
inline std::string stateOutputToStateInputName(const std::string& name) {
|
||||
return READVALUE_PREFIX + name.substr(ASSIGN_PREFIX.length());
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "openvino/runtime/profiling_info.hpp"
|
||||
|
||||
namespace intel_npu::profiling {
|
||||
|
||||
using LayerStatistics = std::vector<ov::ProfilingInfo>;
|
||||
|
||||
template <typename LayerInfo>
|
||||
LayerStatistics convertLayersToIeProfilingInfo(const std::vector<LayerInfo>& layerInfo) {
|
||||
LayerStatistics perfCounts;
|
||||
|
||||
perfCounts.reserve(layerInfo.size());
|
||||
for (const auto& layer : layerInfo) {
|
||||
ov::ProfilingInfo& info = perfCounts.emplace_back();
|
||||
info.status = ov::ProfilingInfo::Status::EXECUTED;
|
||||
const auto real_time_ns = std::chrono::nanoseconds(layer.duration_ns);
|
||||
info.real_time = std::chrono::duration_cast<std::chrono::microseconds>(real_time_ns);
|
||||
const auto cpu_time_ns = std::chrono::nanoseconds(layer.dma_ns + layer.sw_ns + layer.dpu_ns);
|
||||
info.cpu_time = std::chrono::duration_cast<std::chrono::microseconds>(cpu_time_ns);
|
||||
info.node_name = layer.name;
|
||||
if (layer.sw_ns > 0) {
|
||||
info.exec_type = "Shave";
|
||||
} else if (layer.dpu_ns > 0) {
|
||||
info.exec_type = "DPU";
|
||||
} else {
|
||||
info.exec_type = "DMA";
|
||||
}
|
||||
info.node_type = layer.layer_type;
|
||||
}
|
||||
return perfCounts;
|
||||
}
|
||||
|
||||
} // namespace intel_npu::profiling
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "intel_npu/al/config/config.hpp"
|
||||
#include "intel_npu/al/icompiled_model.hpp"
|
||||
#include "intel_npu/al/icompiler.hpp"
|
||||
#include "openvino/runtime/properties.hpp"
|
||||
#include "sync_infer_request.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
class IDevice;
|
||||
|
||||
class IEngineBackend : public std::enable_shared_from_this<IEngineBackend> {
|
||||
public:
|
||||
/** @brief Get device, which can be used for inference. Backend responsible for selection. */
|
||||
virtual const std::shared_ptr<IDevice> getDevice() const;
|
||||
/** @brief Search for a specific device by name */
|
||||
virtual const std::shared_ptr<IDevice> getDevice(const std::string& specificDeviceName) const;
|
||||
/** @brief Get device, which is configured/suitable for provided params */
|
||||
virtual const std::shared_ptr<IDevice> getDevice(const ov::AnyMap& paramMap) const;
|
||||
/** @brief Provide a list of names of all devices, with which user can work directly */
|
||||
virtual const std::vector<std::string> getDeviceNames() const;
|
||||
/** @brief Get name of backend */
|
||||
virtual const std::string getName() const = 0;
|
||||
/** @brief Register backend-specific options */
|
||||
virtual void registerOptions(OptionsDesc& options) const;
|
||||
|
||||
protected:
|
||||
virtual ~IEngineBackend() = default;
|
||||
};
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
class IExecutor {
|
||||
public:
|
||||
virtual ~IExecutor() = default;
|
||||
};
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
class IDevice : public std::enable_shared_from_this<IDevice> {
|
||||
public:
|
||||
using Uuid = ov::device::UUID;
|
||||
|
||||
virtual std::shared_ptr<IExecutor> createExecutor(
|
||||
const std::shared_ptr<const NetworkDescription>& networkDescription,
|
||||
const Config& config) = 0;
|
||||
|
||||
virtual std::string getName() const = 0;
|
||||
virtual std::string getFullDeviceName() const = 0;
|
||||
virtual Uuid getUuid() const;
|
||||
virtual uint32_t getSubDevId() const;
|
||||
virtual uint32_t getMaxNumSlices() const;
|
||||
virtual uint64_t getAllocMemSize() const;
|
||||
virtual uint64_t getTotalMemSize() const;
|
||||
virtual uint32_t getDriverVersion() const;
|
||||
|
||||
virtual std::shared_ptr<SyncInferRequest> createInferRequest(
|
||||
const std::shared_ptr<const ICompiledModel>& compiledModel,
|
||||
const std::shared_ptr<IExecutor>& executor,
|
||||
const Config& config) = 0;
|
||||
|
||||
protected:
|
||||
virtual ~IDevice() = default;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,347 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "openvino/runtime/intel_npu/properties.hpp"
|
||||
|
||||
namespace ov {
|
||||
namespace intel_npu {
|
||||
|
||||
namespace Platform {
|
||||
|
||||
constexpr std::string_view AUTO_DETECT = "AUTO_DETECT"; // Auto detection
|
||||
constexpr std::string_view NPU3700 = "3700"; // NPU30XX
|
||||
constexpr std::string_view NPU3720 = "3720"; // NPU37XX
|
||||
|
||||
/**
|
||||
* @brief Converts the given platform value to the standard one.
|
||||
* @details The same platform value can be defined in multiple ways (e.g. "3720" vs "VPU3720" vs "NPU3720"). The current
|
||||
* function converts the prefixed variants to the non-prefixed ones in order to enable the comparison between platform
|
||||
* values.
|
||||
*
|
||||
* The values already found in the standard form are returned as they are.
|
||||
*
|
||||
* @param platform The value to be converted.
|
||||
* @return The same platform value given as parameter but converted to the standard form.
|
||||
*/
|
||||
inline std::string standardize(const std::string_view platform) {
|
||||
constexpr std::string_view VPUPrefix = "VPU";
|
||||
constexpr std::string_view NPUPrefix = "NPU";
|
||||
|
||||
if (!platform.compare(0, VPUPrefix.length(), VPUPrefix) || !platform.compare(0, NPUPrefix.length(), NPUPrefix)) {
|
||||
return std::string(platform).substr(NPUPrefix.length());
|
||||
}
|
||||
|
||||
return std::string(platform);
|
||||
}
|
||||
|
||||
} // namespace Platform
|
||||
|
||||
/**
|
||||
* @enum ColorFormat
|
||||
* @brief Extra information about input color format for preprocessing
|
||||
* @note Configuration API v 2.0
|
||||
*/
|
||||
enum ColorFormat : uint32_t {
|
||||
RAW = 0u, ///< Plain blob (default), no extra color processing required
|
||||
RGB, ///< RGB color format
|
||||
BGR, ///< BGR color format, default in DLDT
|
||||
RGBX, ///< RGBX color format with X ignored during inference
|
||||
BGRX, ///< BGRX color format with X ignored during inference
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Prints a string representation of ov::intel_npu::ColorFormat to a stream
|
||||
* @param out An output stream to send to
|
||||
* @param fmt A color format value to print to a stream
|
||||
* @return A reference to the `out` stream
|
||||
* @note Configuration API v 2.0
|
||||
*/
|
||||
inline std::ostream& operator<<(std::ostream& out, const ColorFormat& fmt) {
|
||||
switch (fmt) {
|
||||
case ColorFormat::RAW: {
|
||||
out << "RAW";
|
||||
} break;
|
||||
case ColorFormat::RGB: {
|
||||
out << "RGB";
|
||||
} break;
|
||||
case ColorFormat::BGR: {
|
||||
out << "BGR";
|
||||
} break;
|
||||
case ColorFormat::RGBX: {
|
||||
out << "RGBX";
|
||||
} break;
|
||||
case ColorFormat::BGRX: {
|
||||
out << "BGRX";
|
||||
} break;
|
||||
default:
|
||||
out << static_cast<uint32_t>(fmt);
|
||||
break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Plugin]
|
||||
* Type: string, default is MLIR.
|
||||
* Type of NPU compiler to be used for compilation of a network
|
||||
* @note Configuration API v 2.0
|
||||
*/
|
||||
enum class CompilerType { MLIR, DRIVER };
|
||||
|
||||
/**
|
||||
* @brief Prints a string representation of ov::intel_npu::CompilerType to a stream
|
||||
* @param out An output stream to send to
|
||||
* @param fmt A compiler type value to print to a stream
|
||||
* @return A reference to the `out` stream
|
||||
* @note Configuration API v 2.0
|
||||
*/
|
||||
inline std::ostream& operator<<(std::ostream& out, const CompilerType& fmt) {
|
||||
switch (fmt) {
|
||||
case CompilerType::MLIR: {
|
||||
out << "MLIR";
|
||||
} break;
|
||||
case CompilerType::DRIVER: {
|
||||
out << "DRIVER";
|
||||
} break;
|
||||
default:
|
||||
out << static_cast<uint32_t>(fmt);
|
||||
break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Plugin]
|
||||
* Type: String. Default is "AUTO".
|
||||
* This option is added for enabling ELF backend.
|
||||
* Possible values: "AUTO", "YES", "NO".
|
||||
*/
|
||||
|
||||
enum class ElfCompilerBackend {
|
||||
AUTO = 0,
|
||||
NO = 1,
|
||||
YES = 2,
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Prints a string representation of ov::intel_npu::ElfCompilerBackend to a stream
|
||||
* @param out An output stream to send to
|
||||
* @param fmt A elf compiler backend value to print to a stream
|
||||
* @return A reference to the `out` stream
|
||||
* @note Configuration API v 2.0
|
||||
*/
|
||||
inline std::ostream& operator<<(std::ostream& out, const ElfCompilerBackend& fmt) {
|
||||
switch (fmt) {
|
||||
case ElfCompilerBackend::AUTO: {
|
||||
out << "AUTO";
|
||||
} break;
|
||||
case ElfCompilerBackend::NO: {
|
||||
out << "NO";
|
||||
} break;
|
||||
case ElfCompilerBackend::YES: {
|
||||
out << "YES";
|
||||
} break;
|
||||
default:
|
||||
out << static_cast<uint32_t>(fmt);
|
||||
break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Plugin]
|
||||
* Type: string, default is MODEL.
|
||||
* Type of profiling to execute. Can be Model (default) or INFER (based on npu timestamps)
|
||||
* @note Configuration API v 2.0
|
||||
*/
|
||||
enum class ProfilingType { MODEL, INFER };
|
||||
|
||||
/**
|
||||
* @brief Prints a string representation of ov::intel_npu::ProfilingType to a stream
|
||||
* @param out An output stream to send to
|
||||
* @param fmt A profiling type value to print to a stream
|
||||
* @return A reference to the `out` stream
|
||||
* @note Configuration API v 2.0
|
||||
*/
|
||||
inline std::ostream& operator<<(std::ostream& out, const ProfilingType& fmt) {
|
||||
switch (fmt) {
|
||||
case ProfilingType::MODEL: {
|
||||
out << "MODEL";
|
||||
} break;
|
||||
case ProfilingType::INFER: {
|
||||
out << "INFER";
|
||||
} break;
|
||||
default:
|
||||
out << static_cast<uint32_t>(fmt);
|
||||
break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Defines the options corresponding to the legacy set of values.
|
||||
*/
|
||||
enum class LegacyPriority {
|
||||
LOW = 0, //!< Low priority
|
||||
MEDIUM = 1, //!< Medium priority
|
||||
HIGH = 2 //!< High priority
|
||||
};
|
||||
|
||||
inline std::ostream& operator<<(std::ostream& os, const LegacyPriority& priority) {
|
||||
switch (priority) {
|
||||
case LegacyPriority::LOW:
|
||||
return os << "MODEL_PRIORITY_LOW";
|
||||
case LegacyPriority::MEDIUM:
|
||||
return os << "MODEL_PRIORITY_MED";
|
||||
case LegacyPriority::HIGH:
|
||||
return os << "MODEL_PRIORITY_HIGH";
|
||||
default:
|
||||
OPENVINO_THROW("Unsupported model priority value");
|
||||
}
|
||||
}
|
||||
|
||||
inline std::istream& operator>>(std::istream& is, LegacyPriority& priority) {
|
||||
std::string str;
|
||||
is >> str;
|
||||
if (str == "MODEL_PRIORITY_LOW") {
|
||||
priority = LegacyPriority::LOW;
|
||||
} else if (str == "MODEL_PRIORITY_MED") {
|
||||
priority = LegacyPriority::MEDIUM;
|
||||
} else if (str == "MODEL_PRIORITY_HIGH") {
|
||||
priority = LegacyPriority::HIGH;
|
||||
} else {
|
||||
OPENVINO_THROW("Unsupported model priority: ", str);
|
||||
}
|
||||
return is;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Due to driver compatibility constraints, the set of model priority values corresponding to the OpenVINO legacy
|
||||
* API is being maintained here.
|
||||
* @details The OpenVINO API has made changes with regard to the values used to describe model priorities (e.g.
|
||||
* "MODEL_PRIORITY_MED" -> "MEDIUM"). The NPU plugin can't yet discard this since the newer values may not be
|
||||
* recognized by older drivers.
|
||||
*/
|
||||
static constexpr ov::Property<LegacyPriority, ov::PropertyMutability::RO> legacy_model_priority{"MODEL_PRIORITY"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Plugin]
|
||||
* Type: Arbitrary string.
|
||||
* This option allows to specify device.
|
||||
* The plugin accepts any value given through this option. If the device is not available, either the driver or the
|
||||
* compiler will throw an exception depending on the flow running at the time.
|
||||
*/
|
||||
static constexpr ov::Property<std::string> platform{"NPU_PLATFORM"};
|
||||
|
||||
/**
|
||||
* @brief
|
||||
* Type: integer, default is -1
|
||||
* Device stepping ID. If unset, it will be automatically obtained from driver
|
||||
*/
|
||||
static constexpr ov::Property<int64_t> stepping{"NPU_STEPPING"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Plugin]
|
||||
* Type: string, default is DRIVER.
|
||||
* Selects the type of NPU compiler to be used for compilation of a network.
|
||||
* 'DRIVER' is the default value.
|
||||
*/
|
||||
static constexpr ov::Property<CompilerType> compiler_type{"NPU_COMPILER_TYPE"};
|
||||
|
||||
/**
|
||||
* @brief
|
||||
* Selects different compilation pipelines.
|
||||
*/
|
||||
static constexpr ov::Property<std::string> compilation_mode{"NPU_COMPILATION_MODE"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU compiler]
|
||||
* Type: std::string, default is empty.
|
||||
* Sets various parameters supported by the NPU compiler.
|
||||
* Available values: low-precision=true/low-precision=false
|
||||
*/
|
||||
static constexpr ov::Property<std::string> compilation_mode_params{"NPU_COMPILATION_MODE_PARAMS"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Plugin]
|
||||
* Type: integer, default is None
|
||||
* Number of DPU groups
|
||||
*/
|
||||
static constexpr ov::Property<int64_t> dpu_groups{"NPU_DPU_GROUPS"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Compiler]
|
||||
* Type: integer, default is -1
|
||||
* Sets the number of npu tiles that will be used to execute the model. (Replaces NPU_DPU_GROUPS)
|
||||
*/
|
||||
static constexpr ov::Property<int64_t> tiles{"NPU_TILES"};
|
||||
|
||||
/**
|
||||
* @brief
|
||||
* Type: integer, default is -1
|
||||
* Maximum number of tiles supported by the device. If unset, it will be automatically obtained from driver
|
||||
*/
|
||||
static constexpr ov::Property<int64_t> max_tiles{"NPU_MAX_TILES"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Plugin]
|
||||
* Type: integer, default is -1
|
||||
* Sets the number of DMA engines that will be used to execute the model.
|
||||
*/
|
||||
static constexpr ov::Property<int64_t> dma_engines{"NPU_DMA_ENGINES"};
|
||||
|
||||
/**
|
||||
* @brief
|
||||
* Type: Boolean. Default is "NO".
|
||||
* Determines which branch we use for dynamic shapes.
|
||||
* If set to 'YES', we immediately apply the bounds so that we have a static shape for further work.
|
||||
* If not, we store the related information in TensorAttr and the IE representation looks
|
||||
* like this: tensor<1x?x3xf32, {bounds = [1, 18, 3], ..}>.
|
||||
* Possible values: "YES", "NO".
|
||||
*/
|
||||
static constexpr ov::Property<std::string> dynamic_shape_to_static{"NPU_DYNAMIC_SHAPE_TO_STATIC"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Plugin]
|
||||
* Type: string, default is empty.
|
||||
* MODEL - model layer profiling is done
|
||||
* INFER - npu inference performance numbers are measured
|
||||
* Model layers profiling are used if this string is empty
|
||||
*/
|
||||
static constexpr ov::Property<ProfilingType> profiling_type{"NPU_PROFILING_TYPE"};
|
||||
|
||||
/**
|
||||
* @brief
|
||||
* Type: String. Default is "AUTO".
|
||||
* Sets the format in which the compiled model is stored.
|
||||
* Possible values: "AUTO", "YES", "NO".
|
||||
*/
|
||||
static constexpr ov::Property<ElfCompilerBackend> use_elf_compiler_backend{"NPU_USE_ELF_COMPILER_BACKEND"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU Plugin]
|
||||
* Type: integer, default is 1
|
||||
* This option allows to omit creating an executor and therefore to omit running an inference when its value is 0
|
||||
*/
|
||||
static constexpr ov::Property<int64_t> create_executor{"NPU_CREATE_EXECUTOR"};
|
||||
|
||||
/**
|
||||
* @brief Read-only property to get the name of used backend
|
||||
*/
|
||||
static constexpr ov::Property<std::string, ov::PropertyMutability::RO> backend_name{"NPU_BACKEND_NAME"};
|
||||
|
||||
/**
|
||||
* @brief [Only for NPU compiler]
|
||||
* Type: std::string, default is empty.
|
||||
* Config for Backend pipeline
|
||||
|
||||
* Available values: enable-memory-side-cache=true/false
|
||||
* Available values: enable-partial-workload-management=true/false
|
||||
*/
|
||||
static constexpr ov::Property<std::string> backend_compilation_params{"NPU_BACKEND_COMPILATION_PARAMS"};
|
||||
|
||||
} // namespace intel_npu
|
||||
} // namespace ov
|
||||
|
|
@ -0,0 +1,186 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "intel_npu/al/icompiled_model.hpp"
|
||||
#include "intel_npu/al/icompiler.hpp"
|
||||
#include "openvino/runtime/iinfer_request.hpp"
|
||||
#include "openvino/runtime/iplugin.hpp"
|
||||
#include "variable_state.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
/**
|
||||
* @brief Acts as an interface for the inference request structures implemented by all backends.
|
||||
* @details The operations common for all backends can be found implemented here, these include tensors' extraction,
|
||||
* state variables handling and and a few helper functions.
|
||||
* @note Regarding the code design, the "ov::ISyncInferRequest" would normally be the better option for
|
||||
* inheritance. However, the interface exposed by that class forces some additional latency in several unfavorable
|
||||
* scenarios, thus a reimplementation was required.
|
||||
*/
|
||||
class SyncInferRequest : public ov::IInferRequest {
|
||||
public:
|
||||
explicit SyncInferRequest(const std::shared_ptr<const ICompiledModel>& compiledModel);
|
||||
|
||||
/**
|
||||
* @brief Gets an input/output tensor for inference.
|
||||
* @note If the tensor with the specified @p port is not found, an exception is thrown.
|
||||
* @param port Port of the tensor to get.
|
||||
* @return Tensor for the port @p port.
|
||||
*/
|
||||
ov::SoPtr<ov::ITensor> get_tensor(const ov::Output<const ov::Node>& port) const override;
|
||||
|
||||
/**
|
||||
* @brief Sets an input/output tensor to infer.
|
||||
* @param port Port of the input or output tensor.
|
||||
* @param tensor Reference to a tensor. The element_type and shape of a tensor must match
|
||||
* the model's input/output element_type and size.
|
||||
*/
|
||||
void set_tensor(const ov::Output<const ov::Node>& port, const ov::SoPtr<ov::ITensor>& tensor) override;
|
||||
|
||||
/**
|
||||
* @brief Currently there is no support implemented for batches of tensors, thus this call is a simple redirection
|
||||
* to the "get_tensor" one.
|
||||
*/
|
||||
std::vector<ov::SoPtr<ov::ITensor>> get_tensors(const ov::Output<const ov::Node>& port) const override;
|
||||
|
||||
/**
|
||||
* @brief Currently there is no support implemented for batches of tensors, thus this call is a simple redirection
|
||||
* to the "set_tensor" one.
|
||||
*/
|
||||
void set_tensors(const ov::Output<const ov::Node>& port,
|
||||
const std::vector<ov::SoPtr<ov::ITensor>>& tensors) override;
|
||||
|
||||
/**
|
||||
* @brief Gets inputs for infer request
|
||||
*
|
||||
* @return vector of input ports
|
||||
*/
|
||||
const std::vector<ov::Output<const ov::Node>>& get_inputs() const override;
|
||||
|
||||
/**
|
||||
* @brief Gets outputs for infer request
|
||||
*
|
||||
* @return vector of output ports
|
||||
*/
|
||||
const std::vector<ov::Output<const ov::Node>>& get_outputs() const override;
|
||||
|
||||
/**
|
||||
* @brief Gets pointer to compiled model (usually synchronous request holds the compiled model)
|
||||
*
|
||||
* @return Pointer to the compiled model
|
||||
*/
|
||||
const std::shared_ptr<const ov::ICompiledModel>& get_compiled_model() const override;
|
||||
|
||||
/**
|
||||
* @brief Used for executing the inference.
|
||||
*/
|
||||
virtual void infer_async() = 0;
|
||||
|
||||
/**
|
||||
* @brief Used for retrieving the prediction's result.
|
||||
*/
|
||||
virtual void get_result() = 0;
|
||||
|
||||
std::vector<ov::SoPtr<ov::IVariableState>> query_state() const override;
|
||||
|
||||
/**
|
||||
* @brief Initializes the tensor values corresponding to the state variables.
|
||||
* @details The inital values are usually all 0s.
|
||||
*/
|
||||
void initialize_states();
|
||||
|
||||
/**
|
||||
* @return The state tensors accessible by their names.
|
||||
*/
|
||||
std::unordered_map<std::string, std::shared_ptr<VariableState>>& get_variable_states() {
|
||||
return _variableStates;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The names used by the inputs in the order registered inside the model.
|
||||
*/
|
||||
std::vector<std::string> get_input_names() {
|
||||
return _metadata.inputNames;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The names used by the outputs in the order registered inside the model.
|
||||
*/
|
||||
std::vector<std::string> get_output_names() {
|
||||
return _metadata.outputNames;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The names used by the state variables in the order registered inside the model.
|
||||
*/
|
||||
std::vector<std::string> get_state_names() {
|
||||
return _metadata.stateNames;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A map holding references towards all tensors used by the current inference request object.
|
||||
*/
|
||||
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& get_all_tensors() {
|
||||
return _allTensors;
|
||||
}
|
||||
|
||||
protected:
|
||||
/**
|
||||
* @brief Basic checks for input/output tensor
|
||||
*
|
||||
* @param port Input/Output port
|
||||
* @param tensor Input/Output tensor
|
||||
*/
|
||||
void check_tensor(const ov::Output<const ov::Node>& port, const ov::SoPtr<ov::ITensor>& tensor) const;
|
||||
|
||||
/**
|
||||
* @brief Check that all tensors are valid. Throws an exception if it's not.
|
||||
*/
|
||||
void check_tensors() const override;
|
||||
|
||||
/**
|
||||
* @brief Checks if the provided precision value is supported by the current backend, should throw an error
|
||||
* otherwise.
|
||||
* @param precision The precision value to be checked.
|
||||
*/
|
||||
virtual void check_network_precision(const ov::element::Type_t precision) = 0;
|
||||
|
||||
/**
|
||||
* @brief Allocates a tensor on host and stores the reference inside the "_allTensors" attribute. If a buffer
|
||||
* address is provided, then the tensor is built upon it and no additional data buffer is allocated.
|
||||
* @param tensorName The name by which the tensor shall be identified
|
||||
* @param descriptor Tensor's metadata
|
||||
* @param isState If true, the tensor shall also be stored inside the state variables map. In this case, adding the
|
||||
* tensor to this structure would be required in order to correctly answer the state queries.
|
||||
* @param allocator If provided, the tensor uses the custom allocator instead of using the default one.
|
||||
*/
|
||||
void allocate_tensor(std::string tensorName,
|
||||
const IONodeDescriptor& descriptor,
|
||||
const bool isState,
|
||||
const ov::Allocator& allocator = {});
|
||||
|
||||
// Mutable to return reference to ov::Tensor
|
||||
mutable std::unordered_map<std::string, std::shared_ptr<ov::ITensor>> _allTensors;
|
||||
// A copy of each tensor is needed to maintain the original L0 memory allocation in case the user provides another
|
||||
// memory area for the tensor.
|
||||
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>> _copyAllTensors;
|
||||
|
||||
std::unordered_map<std::string, std::shared_ptr<VariableState>> _variableStates;
|
||||
|
||||
// This is intel_npu::ICompiledModel pointer, but need to use OV base class because
|
||||
// ov::IInferRequest::get_compiled_model returns a refernce to shared_ptr!
|
||||
std::shared_ptr<const ov::ICompiledModel> _compiledModel;
|
||||
|
||||
NetworkMetadata _metadata;
|
||||
|
||||
// Stored in order to avoid additional processing when launching inferences
|
||||
std::vector<std::string> _inputAndStateInputNames;
|
||||
std::vector<std::string> _outputAndStateOutputNames;
|
||||
|
||||
std::unordered_map<std::string, std::string> _nodeNameToLegacyName;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "openvino/runtime/itensor.hpp"
|
||||
#include "openvino/runtime/ivariable_state.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
class VariableState final : public ov::IVariableState {
|
||||
public:
|
||||
explicit VariableState(const std::string& name, const std::shared_ptr<ov::ITensor>& tensor)
|
||||
: ov::IVariableState(name) {
|
||||
m_state = tensor;
|
||||
}
|
||||
|
||||
void set_state(const ov::SoPtr<ov::ITensor>& newState) override {
|
||||
if (newState->get_byte_size() != m_state->get_byte_size()) {
|
||||
OPENVINO_THROW("Byte size mismatch");
|
||||
}
|
||||
|
||||
std::memcpy(m_state->data(), newState->data(), newState->get_byte_size());
|
||||
}
|
||||
|
||||
void reset() override {
|
||||
std::memset(m_state->data(), 0, m_state->get_byte_size());
|
||||
}
|
||||
|
||||
~VariableState() override = default;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
|
||||
using namespace intel_npu;
|
||||
using namespace ov::intel_npu;
|
||||
|
||||
//
|
||||
// register
|
||||
//
|
||||
|
||||
void intel_npu::registerCommonOptions(OptionsDesc& desc) {
|
||||
desc.add<PERFORMANCE_HINT>();
|
||||
desc.add<PERFORMANCE_HINT_NUM_REQUESTS>();
|
||||
desc.add<INFERENCE_PRECISION_HINT>();
|
||||
desc.add<PERF_COUNT>();
|
||||
desc.add<LOG_LEVEL>();
|
||||
desc.add<PLATFORM>();
|
||||
desc.add<DEVICE_ID>();
|
||||
desc.add<CACHE_DIR>();
|
||||
desc.add<LOADED_FROM_CACHE>();
|
||||
}
|
||||
|
||||
//
|
||||
// PERFORMANCE_HINT
|
||||
//
|
||||
|
||||
std::string_view ov::hint::stringifyEnum(PerformanceMode val) {
|
||||
switch (val) {
|
||||
case PerformanceMode::LATENCY:
|
||||
return "LATENCY";
|
||||
case PerformanceMode::THROUGHPUT:
|
||||
return "THROUGHPUT";
|
||||
case PerformanceMode::CUMULATIVE_THROUGHPUT:
|
||||
return "CUMULATIVE_THROUGHPUT";
|
||||
default:
|
||||
return "<UNKNOWN>";
|
||||
}
|
||||
}
|
||||
|
||||
ov::hint::PerformanceMode intel_npu::PERFORMANCE_HINT::parse(std::string_view val) {
|
||||
if (val.empty()) {
|
||||
return ov::hint::PerformanceMode::LATENCY;
|
||||
} else if (val == "LATENCY") {
|
||||
return ov::hint::PerformanceMode::LATENCY;
|
||||
} else if (val == "THROUGHPUT") {
|
||||
return ov::hint::PerformanceMode::THROUGHPUT;
|
||||
} else if (val == "CUMULATIVE_THROUGHPUT") {
|
||||
return ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT;
|
||||
}
|
||||
|
||||
OPENVINO_THROW("Value '", val, "' is not a valid PERFORMANCE_HINT option");
|
||||
}
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "intel_npu/al/config/compiler.hpp"
|
||||
|
||||
using namespace intel_npu;
|
||||
using namespace ov::intel_npu;
|
||||
|
||||
//
|
||||
// register
|
||||
//
|
||||
|
||||
void intel_npu::registerCompilerOptions(OptionsDesc& desc) {
|
||||
desc.add<COMPILER_TYPE>();
|
||||
desc.add<COMPILATION_MODE>();
|
||||
desc.add<COMPILATION_MODE_PARAMS>();
|
||||
desc.add<BACKEND_COMPILATION_PARAMS>();
|
||||
desc.add<COMPILATION_NUM_THREADS>();
|
||||
desc.add<DPU_GROUPS>();
|
||||
desc.add<TILES>();
|
||||
desc.add<STEPPING>();
|
||||
desc.add<MAX_TILES>();
|
||||
desc.add<DMA_ENGINES>();
|
||||
desc.add<USE_ELF_COMPILER_BACKEND>();
|
||||
desc.add<DYNAMIC_SHAPE_TO_STATIC>();
|
||||
}
|
||||
|
||||
//
|
||||
// COMPILER_TYPE
|
||||
//
|
||||
|
||||
std::string_view ov::intel_npu::stringifyEnum(ov::intel_npu::CompilerType val) {
|
||||
switch (val) {
|
||||
case ov::intel_npu::CompilerType::MLIR:
|
||||
return "MLIR";
|
||||
case ov::intel_npu::CompilerType::DRIVER:
|
||||
return "DRIVER";
|
||||
default:
|
||||
return "<UNKNOWN>";
|
||||
}
|
||||
}
|
||||
|
||||
std::string_view intel_npu::COMPILER_TYPE::envVar() {
|
||||
#ifdef NPU_PLUGIN_DEVELOPER_BUILD
|
||||
return "IE_NPU_COMPILER_TYPE";
|
||||
#else
|
||||
return "";
|
||||
#endif
|
||||
}
|
||||
|
||||
ov::intel_npu::CompilerType intel_npu::COMPILER_TYPE::defaultValue() {
|
||||
return ov::intel_npu::CompilerType::DRIVER;
|
||||
}
|
||||
|
||||
ov::intel_npu::CompilerType intel_npu::COMPILER_TYPE::parse(std::string_view val) {
|
||||
if (val == stringifyEnum(ov::intel_npu::CompilerType::MLIR)) {
|
||||
return ov::intel_npu::CompilerType::MLIR;
|
||||
} else if (val == stringifyEnum(ov::intel_npu::CompilerType::DRIVER)) {
|
||||
return ov::intel_npu::CompilerType::DRIVER;
|
||||
}
|
||||
|
||||
OPENVINO_THROW("Value '", val, "' is not a valid COMPILER_TYPE option");
|
||||
}
|
||||
|
||||
std::string intel_npu::COMPILER_TYPE::toString(const ov::intel_npu::CompilerType& val) {
|
||||
std::stringstream strStream;
|
||||
if (val == ov::intel_npu::CompilerType::MLIR) {
|
||||
strStream << "MLIR";
|
||||
} else if (val == ov::intel_npu::CompilerType::DRIVER) {
|
||||
strStream << "DRIVER";
|
||||
} else {
|
||||
OPENVINO_THROW("No valid string for current LOG_LEVEL option");
|
||||
}
|
||||
|
||||
return strStream.str();
|
||||
}
|
||||
|
||||
//
|
||||
// USE_ELF_COMPILER_BACKEND
|
||||
//
|
||||
|
||||
std::string_view ov::intel_npu::stringifyEnum(ov::intel_npu::ElfCompilerBackend val) {
|
||||
switch (val) {
|
||||
case ov::intel_npu::ElfCompilerBackend::AUTO:
|
||||
return "AUTO";
|
||||
case ov::intel_npu::ElfCompilerBackend::NO:
|
||||
return "NO";
|
||||
case ov::intel_npu::ElfCompilerBackend::YES:
|
||||
return "YES";
|
||||
default:
|
||||
return "<UNKNOWN>";
|
||||
}
|
||||
}
|
||||
|
||||
ov::intel_npu::ElfCompilerBackend intel_npu::USE_ELF_COMPILER_BACKEND::parse(std::string_view val) {
|
||||
if (val == stringifyEnum(ov::intel_npu::ElfCompilerBackend::AUTO)) {
|
||||
return ov::intel_npu::ElfCompilerBackend::AUTO;
|
||||
} else if (val == stringifyEnum(ov::intel_npu::ElfCompilerBackend::NO)) {
|
||||
return ov::intel_npu::ElfCompilerBackend::NO;
|
||||
} else if (val == stringifyEnum(ov::intel_npu::ElfCompilerBackend::YES)) {
|
||||
return ov::intel_npu::ElfCompilerBackend::YES;
|
||||
}
|
||||
|
||||
OPENVINO_THROW("Value '", val, "' is not a valid USE_ELF_COMPILER_BACKEND option");
|
||||
}
|
||||
|
||||
std::string intel_npu::USE_ELF_COMPILER_BACKEND::toString(const ov::intel_npu::ElfCompilerBackend& val) {
|
||||
std::stringstream strStream;
|
||||
if (val == ov::intel_npu::ElfCompilerBackend::AUTO) {
|
||||
strStream << "AUTO";
|
||||
} else if (val == ov::intel_npu::ElfCompilerBackend::NO) {
|
||||
strStream << "NO";
|
||||
} else if (val == ov::intel_npu::ElfCompilerBackend::YES) {
|
||||
strStream << "YES";
|
||||
} else {
|
||||
OPENVINO_THROW("No valid string for current USE_ELF_COMPILER_BACKEND option");
|
||||
}
|
||||
|
||||
return strStream.str();
|
||||
}
|
||||
|
|
@ -0,0 +1,241 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "intel_npu/al/config/config.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
// Splits the `str` string onto separate elements using `delim` as delimiter and
|
||||
// call `callback` for each element.
|
||||
void splitAndApply(const std::string& str, char delim, std::function<void(std::string_view)> callback) {
|
||||
const auto begin = str.begin();
|
||||
const auto end = str.end();
|
||||
|
||||
auto curBegin = begin;
|
||||
auto curEnd = begin;
|
||||
while (curEnd != end) {
|
||||
while (curEnd != end && *curEnd != delim) {
|
||||
++curEnd;
|
||||
}
|
||||
|
||||
callback(std::string_view(&(*curBegin), static_cast<size_t>(curEnd - curBegin)));
|
||||
|
||||
if (curEnd != end) {
|
||||
++curEnd;
|
||||
curBegin = curEnd;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// OptionParser
|
||||
//
|
||||
|
||||
bool OptionParser<bool>::parse(std::string_view val) {
|
||||
if (val == "YES") {
|
||||
return true;
|
||||
} else if (val == "NO") {
|
||||
return false;
|
||||
}
|
||||
|
||||
OPENVINO_THROW("Value '", val.data(), "' is not a valid BOOL option");
|
||||
}
|
||||
|
||||
int32_t OptionParser<int32_t>::parse(std::string_view val) {
|
||||
try {
|
||||
return std::stol(val.data());
|
||||
} catch (...) {
|
||||
OPENVINO_THROW("Value '%s' is not a valid INT32 option", val.data());
|
||||
}
|
||||
}
|
||||
|
||||
int64_t OptionParser<int64_t>::parse(std::string_view val) {
|
||||
try {
|
||||
return std::stoll(val.data());
|
||||
} catch (...) {
|
||||
OPENVINO_THROW("Value '", val.data(), "' is not a valid INT64 option");
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t OptionParser<uint64_t>::parse(std::string_view val) {
|
||||
try {
|
||||
return std::stoull(val.data());
|
||||
} catch (...) {
|
||||
OPENVINO_THROW("Value '", val.data(), "' is not a valid UINT64 option");
|
||||
}
|
||||
}
|
||||
|
||||
double OptionParser<double>::parse(std::string_view val) {
|
||||
try {
|
||||
return std::stod(val.data());
|
||||
} catch (...) {
|
||||
OPENVINO_THROW("Value '", val.data(), "' is not a valid FP64 option");
|
||||
}
|
||||
}
|
||||
|
||||
ov::log::Level OptionParser<ov::log::Level>::parse(std::string_view val) {
|
||||
std::string strVal(val);
|
||||
std::istringstream is(strVal);
|
||||
ov::log::Level level;
|
||||
is >> level;
|
||||
return level;
|
||||
}
|
||||
|
||||
//
|
||||
// OptionPrinter
|
||||
//
|
||||
|
||||
std::string OptionPrinter<bool>::toString(bool val) {
|
||||
return val ? "YES" : "NO";
|
||||
}
|
||||
|
||||
std::string OptionPrinter<ov::log::Level>::toString(ov::log::Level val) {
|
||||
std::ostringstream os;
|
||||
os << val;
|
||||
return os.str();
|
||||
}
|
||||
|
||||
//
|
||||
// OptionMode
|
||||
//
|
||||
|
||||
std::string_view stringifyEnum(OptionMode val) {
|
||||
switch (val) {
|
||||
case OptionMode::Both:
|
||||
return "Both";
|
||||
case OptionMode::CompileTime:
|
||||
return "CompileTime";
|
||||
case OptionMode::RunTime:
|
||||
return "RunTime";
|
||||
default:
|
||||
return "<UNKNOWN>";
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// OptionValue
|
||||
//
|
||||
|
||||
details::OptionValue::~OptionValue() = default;
|
||||
|
||||
//
|
||||
// OptionsDesc
|
||||
//
|
||||
|
||||
details::OptionConcept OptionsDesc::get(std::string_view key, OptionMode mode) const {
|
||||
auto log = Logger::global().clone("OptionsDesc");
|
||||
|
||||
std::string searchKey{key};
|
||||
const auto itDeprecated = _deprecated.find(std::string(key));
|
||||
if (itDeprecated != _deprecated.end()) {
|
||||
searchKey = itDeprecated->second;
|
||||
log.warning("Deprecated option '%s' was used, '%s' should be used instead", key.data(), searchKey.c_str());
|
||||
}
|
||||
|
||||
const auto itMain = _impl.find(searchKey);
|
||||
OPENVINO_ASSERT(itMain != _impl.end(),
|
||||
"[ NOT_FOUND ] Option '",
|
||||
key.data(),
|
||||
"' is not supported for current configuration");
|
||||
|
||||
const auto& desc = itMain->second;
|
||||
|
||||
if (mode == OptionMode::RunTime) {
|
||||
if (desc.mode() == OptionMode::CompileTime) {
|
||||
log.warning("%s option '%s' was used in %s mode",
|
||||
stringifyEnum(desc.mode()).data(),
|
||||
key.data(),
|
||||
stringifyEnum(mode).data());
|
||||
}
|
||||
}
|
||||
|
||||
return desc;
|
||||
}
|
||||
|
||||
std::vector<std::string> OptionsDesc::getSupported(bool includePrivate) const {
|
||||
std::vector<std::string> res;
|
||||
res.reserve(_impl.size());
|
||||
|
||||
for (const auto& p : _impl) {
|
||||
if (p.second.isPublic() || includePrivate) {
|
||||
res.push_back(p.first);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void OptionsDesc::walk(std::function<void(const details::OptionConcept&)> cb) const {
|
||||
for (const auto& [_, opt] : _impl) {
|
||||
cb(opt);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Config
|
||||
//
|
||||
|
||||
Config::Config(const std::shared_ptr<const OptionsDesc>& desc) : _desc(desc) {
|
||||
OPENVINO_ASSERT(_desc != nullptr, "Got NULL OptionsDesc");
|
||||
}
|
||||
|
||||
void Config::parseEnvVars() {
|
||||
auto log = Logger::global().clone("Config");
|
||||
|
||||
_desc->walk([&](const details::OptionConcept& opt) {
|
||||
if (!opt.envVar().empty()) {
|
||||
if (const auto envVar = std::getenv(opt.envVar().data())) {
|
||||
log.trace("Update option '%s' to value '%s' parsed from environment variable '%s'",
|
||||
opt.key().data(),
|
||||
envVar,
|
||||
opt.envVar().data());
|
||||
|
||||
_impl[opt.key().data()] = opt.validateAndParse(envVar);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void Config::update(const ConfigMap& options, OptionMode mode) {
|
||||
auto log = Logger::global().clone("Config");
|
||||
|
||||
for (const auto& p : options) {
|
||||
log.trace("Update option '%s' to value '%s'", p.first.c_str(), p.second.c_str());
|
||||
|
||||
const auto opt = _desc->get(p.first, mode);
|
||||
_impl[opt.key().data()] = opt.validateAndParse(p.second);
|
||||
}
|
||||
}
|
||||
|
||||
std::string Config::toString() const {
|
||||
std::stringstream resultStream;
|
||||
for (auto it = _impl.cbegin(); it != _impl.cend(); ++it) {
|
||||
const auto key = it->first;
|
||||
|
||||
resultStream << key << "=\"" << it->second->toString() << "\"";
|
||||
if (std::next(it) != _impl.end()) {
|
||||
resultStream << " ";
|
||||
}
|
||||
}
|
||||
|
||||
return resultStream.str();
|
||||
}
|
||||
|
||||
//
|
||||
// envVarStrToBool
|
||||
//
|
||||
|
||||
bool envVarStrToBool(const char* varName, const char* varValue) {
|
||||
try {
|
||||
const auto intVal = std::stoi(varValue);
|
||||
if (intVal != 0 && intVal != 1) {
|
||||
throw std::invalid_argument("Only 0 and 1 values are supported");
|
||||
}
|
||||
return (intVal != 0);
|
||||
} catch (const std::exception& e) {
|
||||
OPENVINO_THROW(std::string("Environment variable ") + varName + " has wrong value : " + e.what());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "intel_npu/al/config/runtime.hpp"
|
||||
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
|
||||
using namespace intel_npu;
|
||||
using namespace ov::intel_npu;
|
||||
|
||||
//
|
||||
// register
|
||||
//
|
||||
|
||||
void intel_npu::registerRunTimeOptions(OptionsDesc& desc) {
|
||||
desc.add<EXCLUSIVE_ASYNC_REQUESTS>();
|
||||
desc.add<PROFILING_TYPE>();
|
||||
desc.add<MODEL_PRIORITY>();
|
||||
desc.add<CREATE_EXECUTOR>();
|
||||
desc.add<NUM_STREAMS>();
|
||||
desc.add<ENABLE_CPU_PINNING>();
|
||||
}
|
||||
|
||||
// Heuristically obtained number. Varies depending on the values of PLATFORM and PERFORMANCE_HINT
|
||||
// Note: this is the value provided by the plugin, application should query and consider it, but may supply its own
|
||||
// preference for number of parallel requests via dedicated configuration
|
||||
int64_t intel_npu::getOptimalNumberOfInferRequestsInParallel(const Config& config) {
|
||||
const std::string platform = ov::intel_npu::Platform::standardize(config.get<PLATFORM>());
|
||||
|
||||
if (platform == ov::intel_npu::Platform::NPU3720) {
|
||||
if (config.get<PERFORMANCE_HINT>() == ov::hint::PerformanceMode::THROUGHPUT) {
|
||||
return 4;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
if (config.get<PERFORMANCE_HINT>() == ov::hint::PerformanceMode::THROUGHPUT) {
|
||||
return 8;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// PROFILING_TYPE
|
||||
//
|
||||
|
||||
std::string_view ov::intel_npu::stringifyEnum(ov::intel_npu::ProfilingType val) {
|
||||
switch (val) {
|
||||
case ov::intel_npu::ProfilingType::MODEL:
|
||||
return "MODEL";
|
||||
case ov::intel_npu::ProfilingType::INFER:
|
||||
return "INFER";
|
||||
default:
|
||||
return "<UNKNOWN>";
|
||||
}
|
||||
}
|
||||
|
||||
ov::intel_npu::ProfilingType intel_npu::PROFILING_TYPE::parse(std::string_view val) {
|
||||
const auto extractProfilingString = [](ov::intel_npu::ProfilingType prof) -> std::string {
|
||||
return profiling_type(prof).second.as<std::string>();
|
||||
};
|
||||
|
||||
if (val == extractProfilingString(ov::intel_npu::ProfilingType::MODEL)) {
|
||||
return ov::intel_npu::ProfilingType::MODEL;
|
||||
} else if (val == extractProfilingString(ov::intel_npu::ProfilingType::INFER)) {
|
||||
return ov::intel_npu::ProfilingType::INFER;
|
||||
}
|
||||
|
||||
OPENVINO_THROW("Value '", val, "' is not a valid PROFILING_TYPE option");
|
||||
}
|
||||
|
||||
std::string intel_npu::PROFILING_TYPE::toString(const ov::intel_npu::ProfilingType& val) {
|
||||
std::stringstream strStream;
|
||||
if (val == ov::intel_npu::ProfilingType::MODEL) {
|
||||
strStream << "MODEL";
|
||||
} else if (val == ov::intel_npu::ProfilingType::INFER) {
|
||||
strStream << "INFER";
|
||||
} else {
|
||||
OPENVINO_THROW("No valid string for current PROFILING_TYPE option");
|
||||
}
|
||||
|
||||
return strStream.str();
|
||||
}
|
||||
|
||||
//
|
||||
// MODEL_PRIORITY
|
||||
//
|
||||
|
||||
ov::hint::Priority intel_npu::MODEL_PRIORITY::parse(std::string_view val) {
|
||||
std::istringstream stringStream = std::istringstream(std::string(val));
|
||||
ov::hint::Priority priority;
|
||||
|
||||
stringStream >> priority;
|
||||
|
||||
return priority;
|
||||
}
|
||||
|
||||
std::string intel_npu::MODEL_PRIORITY::toString(const ov::hint::Priority& val) {
|
||||
std::ostringstream stringStream;
|
||||
|
||||
stringStream << val;
|
||||
|
||||
return stringStream.str();
|
||||
}
|
||||
|
||||
//
|
||||
// NUM_STREAMS
|
||||
//
|
||||
|
||||
const ov::streams::Num intel_npu::NUM_STREAMS::defVal = ov::streams::Num(1);
|
||||
|
||||
ov::streams::Num intel_npu::NUM_STREAMS::parse(std::string_view val) {
|
||||
std::istringstream stringStream = std::istringstream(std::string(val));
|
||||
ov::streams::Num numberOfStreams;
|
||||
|
||||
stringStream >> numberOfStreams;
|
||||
|
||||
return numberOfStreams;
|
||||
}
|
||||
|
||||
std::string intel_npu::NUM_STREAMS::toString(const ov::streams::Num& val) {
|
||||
std::ostringstream stringStream;
|
||||
|
||||
stringStream << val;
|
||||
|
||||
return stringStream.str();
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "device_helpers.hpp"
|
||||
|
||||
#include "openvino/core/except.hpp"
|
||||
|
||||
bool utils::isNPUDevice(const uint32_t deviceId) {
|
||||
// bits 26-24 define interface type
|
||||
// 000 - IPC
|
||||
// 001 - PCIe
|
||||
// 010 - USB
|
||||
// 011 - ethernet
|
||||
constexpr uint32_t INTERFACE_TYPE_SELECTOR = 0x7000000;
|
||||
uint32_t interfaceType = (deviceId & INTERFACE_TYPE_SELECTOR);
|
||||
return (interfaceType == 0);
|
||||
}
|
||||
|
||||
uint32_t utils::getSliceIdBySwDeviceId(const uint32_t swDevId) {
|
||||
// bits 3-1 define slice ID
|
||||
// right shift to omit bit 0, thus slice id is stored in bits 2-0
|
||||
// apply b111 mask to discard anything but slice ID
|
||||
uint32_t sliceId = (swDevId >> 1) & 0x7;
|
||||
return sliceId;
|
||||
}
|
||||
|
||||
std::string utils::getPlatformByDeviceName(const std::string& deviceName) {
|
||||
const auto platformPos = deviceName.rfind('.');
|
||||
const auto platformName = (platformPos == std::string::npos) ? deviceName : deviceName.substr(0, platformPos);
|
||||
|
||||
return platformName;
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "npu.hpp"
|
||||
|
||||
#include "intel_npu/al/itt.hpp"
|
||||
#include "openvino/util/shared_object.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
const std::shared_ptr<IDevice> IEngineBackend::getDevice() const {
|
||||
OPENVINO_THROW("Default getDevice() not implemented");
|
||||
}
|
||||
const std::shared_ptr<IDevice> IEngineBackend::getDevice(const std::string&) const {
|
||||
OPENVINO_THROW("Specific device search not implemented");
|
||||
}
|
||||
const std::shared_ptr<IDevice> IEngineBackend::getDevice(const ov::AnyMap&) const {
|
||||
OPENVINO_THROW("Get device based on params not implemented");
|
||||
}
|
||||
const std::vector<std::string> IEngineBackend::getDeviceNames() const {
|
||||
OPENVINO_THROW("Get all device names not implemented");
|
||||
}
|
||||
|
||||
void IEngineBackend::registerOptions(OptionsDesc&) const {}
|
||||
|
||||
IDevice::Uuid IDevice::getUuid() const {
|
||||
OPENVINO_THROW("Get UUID not supported");
|
||||
}
|
||||
|
||||
uint32_t IDevice::getSubDevId() const {
|
||||
OPENVINO_THROW("Get SubDevId is not supported");
|
||||
}
|
||||
|
||||
uint32_t IDevice::getMaxNumSlices() const {
|
||||
OPENVINO_THROW("Get MaxNumSlices is not supported");
|
||||
}
|
||||
|
||||
uint64_t IDevice::getAllocMemSize() const {
|
||||
OPENVINO_THROW("Get AllocMemSize is not supported");
|
||||
}
|
||||
|
||||
uint64_t IDevice::getTotalMemSize() const {
|
||||
OPENVINO_THROW("Get TotalMemSize is not supported");
|
||||
}
|
||||
|
||||
uint32_t IDevice::getDriverVersion() const {
|
||||
OPENVINO_THROW("Get NPU driver version is not supported with this backend");
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,186 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "sync_infer_request.hpp"
|
||||
|
||||
#include "intel_npu/al/prefix.hpp"
|
||||
#include "openvino/runtime/make_tensor.hpp"
|
||||
#include "openvino/runtime/plugin_itt.hpp"
|
||||
#include "transformations/utils/utils.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
SyncInferRequest::SyncInferRequest(const std::shared_ptr<const ICompiledModel>& compiledModel)
|
||||
: _compiledModel(compiledModel),
|
||||
_metadata(compiledModel->get_network_metadata()) {
|
||||
OPENVINO_ASSERT(_compiledModel);
|
||||
|
||||
const std::vector<ov::Output<const ov::Node>>& inputs = get_inputs();
|
||||
const std::vector<ov::Output<const ov::Node>>& outputs = get_outputs();
|
||||
|
||||
if (inputs.empty()) {
|
||||
OPENVINO_THROW("Inference request creation: no input found for network " + _metadata.name);
|
||||
}
|
||||
if (outputs.empty()) {
|
||||
OPENVINO_THROW("Inference request creation: no output found for network " + _metadata.name);
|
||||
}
|
||||
|
||||
// Map the node names to the legacy ones used by the I/O tensors in order to allow an easier access to the tensors'
|
||||
// contents
|
||||
for (const auto& [legacyName, parameterDescriptor] : _metadata.parameters) {
|
||||
_nodeNameToLegacyName[parameterDescriptor.currentNodeName] = legacyName;
|
||||
}
|
||||
for (const auto& [legacyName, resultDescriptor] : _metadata.results) {
|
||||
_nodeNameToLegacyName[resultDescriptor.currentNodeName] = legacyName;
|
||||
}
|
||||
|
||||
_inputAndStateInputNames = _metadata.inputNames;
|
||||
_outputAndStateOutputNames = _metadata.outputNames;
|
||||
|
||||
for (const std::string& stateName : _metadata.stateNames) {
|
||||
// State variables shall be identified by specific prefixes in order to avoid a potential tensor name collision
|
||||
_inputAndStateInputNames.push_back(READVALUE_PREFIX + stateName);
|
||||
_outputAndStateOutputNames.push_back(ASSIGN_PREFIX + stateName);
|
||||
}
|
||||
}
|
||||
|
||||
const std::vector<ov::Output<const ov::Node>>& SyncInferRequest::get_inputs() const {
|
||||
return _compiledModel->inputs();
|
||||
}
|
||||
|
||||
const std::vector<ov::Output<const ov::Node>>& SyncInferRequest::get_outputs() const {
|
||||
return _compiledModel->outputs();
|
||||
}
|
||||
|
||||
const std::shared_ptr<const ov::ICompiledModel>& SyncInferRequest::get_compiled_model() const {
|
||||
return _compiledModel;
|
||||
}
|
||||
|
||||
void SyncInferRequest::initialize_states() {
|
||||
for (const std::string& stateName : _metadata.stateNames) {
|
||||
_variableStates.at(stateName)->reset();
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<ov::SoPtr<ov::IVariableState>> SyncInferRequest::query_state() const {
|
||||
std::vector<ov::SoPtr<ov::IVariableState>> queryResult;
|
||||
|
||||
for (const std::string& stateName : _metadata.stateNames) {
|
||||
queryResult.push_back(_variableStates.at(stateName));
|
||||
}
|
||||
|
||||
return queryResult;
|
||||
}
|
||||
|
||||
ov::SoPtr<ov::ITensor> SyncInferRequest::get_tensor(const ov::Output<const ov::Node>& port) const {
|
||||
const auto& nodeNameMatch = _nodeNameToLegacyName.find(port.get_node()->get_friendly_name());
|
||||
OPENVINO_ASSERT(nodeNameMatch != _nodeNameToLegacyName.end(), "Cannot find tensor for port ", port);
|
||||
|
||||
return _allTensors.at(nodeNameMatch->second);
|
||||
}
|
||||
|
||||
void SyncInferRequest::set_tensor(const ov::Output<const ov::Node>& port, const ov::SoPtr<ov::ITensor>& tensor) {
|
||||
OV_ITT_SCOPED_TASK(ov::itt::domains::Plugin, "set_tensor");
|
||||
try {
|
||||
check_tensor(port, tensor);
|
||||
} catch (const ov::Exception& ex) {
|
||||
OPENVINO_THROW("Failed to set tensor. ", ex.what());
|
||||
}
|
||||
|
||||
const std::string& legacyName = _nodeNameToLegacyName.at(port.get_node()->get_friendly_name());
|
||||
_allTensors[legacyName] = tensor._ptr;
|
||||
}
|
||||
|
||||
std::vector<ov::SoPtr<ov::ITensor>> SyncInferRequest::get_tensors(const ov::Output<const ov::Node>& /*port*/) const {
|
||||
OV_ITT_SCOPED_TASK(ov::itt::domains::Plugin, "get_tensors");
|
||||
// Using batches of tensors is currently not supported by the NPU plugin. In this scenario, the OpenVINO API demands
|
||||
// returning an empty vector.
|
||||
return {};
|
||||
}
|
||||
|
||||
void SyncInferRequest::set_tensors(const ov::Output<const ov::Node>& port,
|
||||
const std::vector<ov::SoPtr<ov::ITensor>>& tensors) {
|
||||
OV_ITT_SCOPED_TASK(ov::itt::domains::Plugin, "set_tensors");
|
||||
if (tensors.size() == 1) {
|
||||
set_tensor(port, tensors[0]);
|
||||
return;
|
||||
}
|
||||
|
||||
OPENVINO_THROW_NOT_IMPLEMENTED("set_input_tensors/set_tensors are not supported by this plugin");
|
||||
}
|
||||
|
||||
void SyncInferRequest::check_tensor(const ov::Output<const ov::Node>& port,
|
||||
const ov::SoPtr<ov::ITensor>& tensor) const {
|
||||
if (tensor == nullptr)
|
||||
OPENVINO_THROW("The tensor is not initialized!");
|
||||
|
||||
bool is_input = ov::op::util::is_parameter(port.get_node());
|
||||
std::string tensor_type = is_input ? "input" : "output";
|
||||
|
||||
OPENVINO_ASSERT(port.get_element_type() == tensor->get_element_type(),
|
||||
"The tensor element type is not corresponding with output element type (",
|
||||
tensor->get_element_type(),
|
||||
" != ",
|
||||
port.get_element_type());
|
||||
bool is_dynamic = port.get_partial_shape().is_dynamic();
|
||||
OPENVINO_ASSERT(is_dynamic || port.get_shape() == tensor->get_shape(),
|
||||
"The ",
|
||||
tensor_type,
|
||||
" tensor size is not equal to the model ",
|
||||
tensor_type,
|
||||
" type: got ",
|
||||
tensor->get_shape(),
|
||||
" expecting ",
|
||||
port.get_shape(),
|
||||
".");
|
||||
OPENVINO_ASSERT(
|
||||
std::dynamic_pointer_cast<ov::IRemoteTensor>(tensor._ptr) || tensor->data() != nullptr || is_dynamic,
|
||||
"Tensor data equal nullptr!");
|
||||
}
|
||||
|
||||
void SyncInferRequest::check_tensors() const {
|
||||
const auto& inputs = _compiledModel->inputs();
|
||||
for (size_t i = 0; i < inputs.size(); i++) {
|
||||
const std::string& legacyName = _nodeNameToLegacyName.at(inputs[i].get_node()->get_friendly_name());
|
||||
check_tensor(inputs[i], _allTensors.at(legacyName));
|
||||
}
|
||||
|
||||
const auto& outputs = _compiledModel->outputs();
|
||||
for (size_t i = 0; i < outputs.size(); i++) {
|
||||
const std::string& legacyName = _nodeNameToLegacyName.at(outputs[i].get_node()->get_friendly_name());
|
||||
check_tensor(outputs[i], _allTensors.at(legacyName));
|
||||
}
|
||||
}
|
||||
|
||||
void SyncInferRequest::allocate_tensor(std::string tensorName,
|
||||
const IONodeDescriptor& descriptor,
|
||||
const bool isState,
|
||||
const ov::Allocator& allocator) {
|
||||
std::shared_ptr<ov::ITensor> tensor;
|
||||
|
||||
check_network_precision(descriptor.precision);
|
||||
|
||||
if (allocator) {
|
||||
tensor = ov::make_tensor(descriptor.precision, descriptor.transposedShape.get_shape(), allocator);
|
||||
} else {
|
||||
tensor = ov::make_tensor(descriptor.precision, descriptor.transposedShape.get_shape());
|
||||
}
|
||||
|
||||
if (!isState) {
|
||||
_copyAllTensors[tensorName] = std::move(tensor);
|
||||
_allTensors[tensorName] = _copyAllTensors[tensorName];
|
||||
} else {
|
||||
_variableStates[tensorName] = std::make_shared<VariableState>(tensorName, tensor);
|
||||
|
||||
// State variables shall be identified by specific prefixes in order to avoid a potential tensor name collision.
|
||||
// Additionally, only one buffer is required in the whole flow, acting as an input before running the inference
|
||||
// and as an output after performing it. Thus both the "state input" and "state output" entries shall point to
|
||||
// the same buffer.
|
||||
_copyAllTensors[READVALUE_PREFIX + tensorName] = std::move(tensor);
|
||||
_copyAllTensors[ASSIGN_PREFIX + tensorName] = _copyAllTensors[READVALUE_PREFIX + tensorName];
|
||||
_allTensors[READVALUE_PREFIX + tensorName] = _copyAllTensors[READVALUE_PREFIX + tensorName];
|
||||
_allTensors[ASSIGN_PREFIX + tensorName] = _copyAllTensors[READVALUE_PREFIX + tensorName];
|
||||
}
|
||||
}
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set(TARGET_NAME "openvino_npu_level_zero_backend")
|
||||
|
||||
file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h)
|
||||
source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES})
|
||||
|
||||
add_library(${TARGET_NAME} STATIC ${SOURCES})
|
||||
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
|
||||
|
||||
target_compile_definitions(${TARGET_NAME}
|
||||
PRIVATE
|
||||
IMPLEMENT_OPENVINO_RUNTIME_PLUGIN)
|
||||
|
||||
target_include_directories(${TARGET_NAME}
|
||||
PUBLIC
|
||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
|
||||
)
|
||||
|
||||
target_link_libraries(${TARGET_NAME}
|
||||
PRIVATE
|
||||
openvino::npu_al
|
||||
openvino_npu_zero_result_parser
|
||||
ze_loader
|
||||
)
|
||||
|
||||
#
|
||||
# targets install
|
||||
#
|
||||
ov_install_static_lib(${TARGET_NAME} ${NPU_INTERNAL_COMPONENT})
|
||||
|
||||
if(TARGET ze_loader)
|
||||
if(NOT BUILD_SHARED_LIBS)
|
||||
# Support link of static runtime in case system does not have ze_loader
|
||||
install(TARGETS ze_loader EXPORT OpenVINOTargets
|
||||
RUNTIME DESTINATION ${OV_CPACK_RUNTIMEDIR} COMPONENT ${NPU_PLUGIN_COMPONENT}
|
||||
ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR} COMPONENT ${NPU_PLUGIN_COMPONENT}
|
||||
LIBRARY DESTINATION ${OV_CPACK_LIBRARYDIR} COMPONENT ${NPU_PLUGIN_COMPONENT})
|
||||
|
||||
# export to local tree to build against static build tree
|
||||
export(TARGETS ze_loader NAMESPACE openvino::
|
||||
APPEND FILE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake")
|
||||
endif()
|
||||
|
||||
# Support tests to run with ze_loader
|
||||
install(TARGETS ze_loader
|
||||
LIBRARY DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL)
|
||||
endif()
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
|
||||
#include "npu.hpp"
|
||||
#include "zero_init.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
class ZeroEngineBackend final : public IEngineBackend {
|
||||
public:
|
||||
ZeroEngineBackend(const Config& config);
|
||||
virtual ~ZeroEngineBackend();
|
||||
const std::shared_ptr<IDevice> getDevice() const override;
|
||||
const std::shared_ptr<IDevice> getDevice(const std::string&) const override;
|
||||
const std::string getName() const override {
|
||||
return "LEVEL0";
|
||||
}
|
||||
const std::vector<std::string> getDeviceNames() const override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<ZeroInitStructsHolder> _instance;
|
||||
|
||||
std::map<std::string, std::shared_ptr<IDevice>> _devices{};
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include "intel_npu/al/icompiled_model.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "npu.hpp"
|
||||
#include "zero_init.hpp"
|
||||
#include "zero_types.hpp"
|
||||
#include "zero_utils.hpp"
|
||||
namespace intel_npu {
|
||||
|
||||
class ZeroDevice : public IDevice {
|
||||
public:
|
||||
ZeroDevice(const std::shared_ptr<ZeroInitStructsHolder>& initStructs);
|
||||
|
||||
std::shared_ptr<IExecutor> createExecutor(const std::shared_ptr<const NetworkDescription>& networkDescription,
|
||||
const Config& config) override;
|
||||
|
||||
std::string getName() const override;
|
||||
std::string getFullDeviceName() const override;
|
||||
Uuid getUuid() const override;
|
||||
uint32_t getSubDevId() const override;
|
||||
uint32_t getMaxNumSlices() const override;
|
||||
uint64_t getAllocMemSize() const override;
|
||||
uint64_t getTotalMemSize() const override;
|
||||
uint32_t getDriverVersion() const override;
|
||||
|
||||
std::shared_ptr<SyncInferRequest> createInferRequest(const std::shared_ptr<const ICompiledModel>& compiledModel,
|
||||
const std::shared_ptr<IExecutor>& executor,
|
||||
const Config& config) override;
|
||||
|
||||
ZeroDevice& operator=(const ZeroDevice&) = delete;
|
||||
ZeroDevice(const ZeroDevice&) = delete;
|
||||
|
||||
private:
|
||||
const std::shared_ptr<ZeroInitStructsHolder> _initStructs;
|
||||
|
||||
ze_graph_dditable_ext_curr_t* _graph_ddi_table_ext = nullptr;
|
||||
|
||||
ze_device_properties_t device_properties = {};
|
||||
ze_driver_properties_t driver_properties = {};
|
||||
|
||||
uint32_t _group_ordinal;
|
||||
|
||||
Logger log;
|
||||
};
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "npu.hpp"
|
||||
#include "zero_init.hpp"
|
||||
#include "zero_wrappers.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
class ZeroExecutor final : public IExecutor {
|
||||
public:
|
||||
ZeroExecutor(const std::shared_ptr<const ZeroInitStructsHolder>& initStructs,
|
||||
const std::shared_ptr<const NetworkDescription>& networkDescription,
|
||||
const Config& config,
|
||||
const uint32_t& group_ordinal);
|
||||
|
||||
ZeroExecutor(const ZeroExecutor&) = delete;
|
||||
ZeroExecutor& operator=(const ZeroExecutor&) = delete;
|
||||
|
||||
~ZeroExecutor() override;
|
||||
|
||||
struct ArgumentDescriptor {
|
||||
ze_graph_argument_properties_t info;
|
||||
uint32_t idx;
|
||||
};
|
||||
|
||||
void setArgumentValue(uint32_t argi_, const void* argv_) const;
|
||||
inline ze_graph_handle_t graph() const {
|
||||
return _graph;
|
||||
}
|
||||
inline std::shared_ptr<const ZeroInitStructsHolder> getInitStructs() const {
|
||||
return _initStructs;
|
||||
}
|
||||
inline const std::shared_ptr<const NetworkDescription>& getNetworkDesc() const {
|
||||
return _networkDesc;
|
||||
}
|
||||
inline const std::array<std::shared_ptr<CommandQueue>, stage::COUNT>& getCommandQueue() const {
|
||||
return _command_queues;
|
||||
}
|
||||
inline const uint32_t& get_group_ordinal() const {
|
||||
return _group_ordinal;
|
||||
}
|
||||
inline const std::unordered_map<std::string, ArgumentDescriptor>& inputs_desc_map() const {
|
||||
return _inputs_desc_map;
|
||||
}
|
||||
inline const std::unordered_map<std::string, ArgumentDescriptor>& outputs_desc_map() const {
|
||||
return _outputs_desc_map;
|
||||
}
|
||||
|
||||
private:
|
||||
const Config _config;
|
||||
Logger _logger;
|
||||
|
||||
const std::shared_ptr<const ZeroInitStructsHolder> _initStructs;
|
||||
std::shared_ptr<const NetworkDescription> _networkDesc;
|
||||
|
||||
ze_graph_dditable_ext_curr_t* _graph_ddi_table_ext = nullptr;
|
||||
|
||||
const uint32_t _group_ordinal;
|
||||
|
||||
ze_graph_handle_t _graph = nullptr;
|
||||
ze_graph_properties_t _props{};
|
||||
std::unordered_map<std::string, ArgumentDescriptor> _inputs_desc_map;
|
||||
std::unordered_map<std::string, ArgumentDescriptor> _outputs_desc_map;
|
||||
|
||||
std::array<std::shared_ptr<CommandQueue>, stage::COUNT> _command_queues;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include <mutex>
|
||||
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "npu.hpp"
|
||||
#include "zero_executor.hpp"
|
||||
#include "zero_pipeline.hpp"
|
||||
#include "zero_profiling.hpp"
|
||||
#include "zero_utils.hpp"
|
||||
#include "zero_wrappers.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
class ZeroInferRequest final : public SyncInferRequest {
|
||||
public:
|
||||
explicit ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>& backendPtr,
|
||||
const std::shared_ptr<const ICompiledModel>& compiledModel,
|
||||
const std::shared_ptr<const IExecutor>& executor,
|
||||
const Config& config);
|
||||
|
||||
void infer() override;
|
||||
void infer_async() override;
|
||||
|
||||
void get_result() override;
|
||||
|
||||
private:
|
||||
std::vector<ov::ProfilingInfo> get_profiling_info() const override;
|
||||
std::vector<uint8_t> get_raw_profiling_data() const;
|
||||
|
||||
void check_network_precision(const ov::element::Type_t precision) override;
|
||||
|
||||
const std::shared_ptr<const IExecutor> _executorPtr;
|
||||
const ZeroExecutor* _executor;
|
||||
const Config _config;
|
||||
Logger _logger;
|
||||
|
||||
zeroProfiling::ProfilingPool _profiling_pool;
|
||||
zeroProfiling::ProfilingQuery _profiling_query;
|
||||
std::shared_ptr<zeroProfiling::NpuInferProfiling> _npu_profiling;
|
||||
std::unique_ptr<Pipeline> _pipeline;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "ze_intel_vpu_uuid.h"
|
||||
#include "zero_types.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
/**
|
||||
* Holder for the level zero structures which must be initialized via call to the driver once zero backend is loaded,
|
||||
* and de-initialized after their last use is over.
|
||||
*/
|
||||
class ZeroInitStructsHolder final {
|
||||
public:
|
||||
ZeroInitStructsHolder();
|
||||
|
||||
ZeroInitStructsHolder(const ZeroInitStructsHolder&) = delete;
|
||||
ZeroInitStructsHolder& operator=(const ZeroInitStructsHolder&) = delete;
|
||||
|
||||
~ZeroInitStructsHolder();
|
||||
|
||||
inline ze_driver_handle_t getDriver() const {
|
||||
return driver_handle;
|
||||
}
|
||||
inline ze_device_handle_t getDevice() const {
|
||||
return device_handle;
|
||||
}
|
||||
inline ze_context_handle_t getContext() const {
|
||||
return context;
|
||||
}
|
||||
inline ze_graph_dditable_ext_curr_t* getGraphDdiTable() const {
|
||||
return graph_dditable_ext_decorator.get();
|
||||
}
|
||||
inline ze_graph_profiling_dditable_ext_t* getProfilingDdiTable() const {
|
||||
return _graph_profiling_ddi_table_ext;
|
||||
}
|
||||
|
||||
private:
|
||||
static const ze_driver_uuid_t uuid;
|
||||
Logger log;
|
||||
|
||||
ze_driver_handle_t driver_handle = nullptr;
|
||||
ze_device_handle_t device_handle = nullptr;
|
||||
ze_context_handle_t context = nullptr;
|
||||
std::unique_ptr<ze_graph_dditable_ext_decorator> graph_dditable_ext_decorator;
|
||||
ze_graph_profiling_dditable_ext_t* _graph_profiling_ddi_table_ext = nullptr;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "zero_init.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::size_t STANDARD_PAGE_SIZE = 4096;
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace intel_npu {
|
||||
namespace zeroMemory {
|
||||
struct DeviceMem {
|
||||
DeviceMem() = delete;
|
||||
DeviceMem(const ze_device_handle_t device_handle, const ze_context_handle_t context, const std::size_t size);
|
||||
DeviceMem(const DeviceMem&) = delete;
|
||||
DeviceMem(DeviceMem&& other)
|
||||
: _size(other._size),
|
||||
_data(other._data),
|
||||
_context(other._context),
|
||||
_log("DeviceMem", Logger::global().level()) {
|
||||
other._size = 0;
|
||||
other._data = nullptr;
|
||||
}
|
||||
DeviceMem& operator=(const DeviceMem&) = delete;
|
||||
DeviceMem& operator=(DeviceMem&& other);
|
||||
|
||||
const void* data() const {
|
||||
return _data;
|
||||
}
|
||||
void* data() {
|
||||
return _data;
|
||||
}
|
||||
std::size_t size() const {
|
||||
return _size;
|
||||
}
|
||||
void free();
|
||||
~DeviceMem();
|
||||
|
||||
private:
|
||||
std::size_t _size = 0;
|
||||
void* _data = nullptr;
|
||||
ze_context_handle_t _context = nullptr;
|
||||
static const std::size_t _alignment = STANDARD_PAGE_SIZE;
|
||||
|
||||
Logger _log;
|
||||
};
|
||||
|
||||
// Create an allocator that uses the ov::Allocator signature that will be used to create the tensor.
|
||||
class HostMemAllocator final {
|
||||
public:
|
||||
explicit HostMemAllocator(const std::shared_ptr<ZeroInitStructsHolder>& initStructs,
|
||||
ze_host_mem_alloc_flag_t flag = {})
|
||||
: _initStructs(initStructs),
|
||||
_flag(flag) {}
|
||||
|
||||
/**
|
||||
* @brief Allocates memory
|
||||
* @param bytes The size in bytes to allocate
|
||||
* @return Handle to the allocated resource
|
||||
*/
|
||||
void* allocate(const size_t bytes, const size_t alignment = STANDARD_PAGE_SIZE) noexcept;
|
||||
/**
|
||||
* @brief Releases handle and all associated memory resources which invalidates the handle.
|
||||
* @param handle Pointer to allocated data
|
||||
* @return false if handle cannot be released, otherwise - true.
|
||||
*/
|
||||
bool deallocate(void* handle, const size_t bytes, size_t alignment = STANDARD_PAGE_SIZE) noexcept;
|
||||
|
||||
bool is_equal(const HostMemAllocator& other) const;
|
||||
|
||||
private:
|
||||
const std::shared_ptr<ZeroInitStructsHolder> _initStructs;
|
||||
|
||||
ze_host_mem_alloc_flag_t _flag;
|
||||
void* _data = nullptr;
|
||||
static const std::size_t _alignment = STANDARD_PAGE_SIZE;
|
||||
};
|
||||
|
||||
// For graph arguments (inputs and outputs) memory should be located on a host side. For discrete HW
|
||||
// generation the arguments has to be moved to device side to make it accessible.
|
||||
// MemoryManagementUnit allow to keeps device allocations in case of discrete HW.
|
||||
// Usage: we should append graph arguments with corresponding names with `appendArgument` call
|
||||
// to prepare size statistics and lookup table. To commit memory allocation we should call `allocate`
|
||||
struct MemoryManagementUnit {
|
||||
MemoryManagementUnit() = default;
|
||||
|
||||
void appendArgument(const std::string& name, const ze_graph_argument_properties_t& argument);
|
||||
/* Allocate Device memories */
|
||||
void allocate(const ze_device_handle_t device_handle, const ze_context_handle_t context);
|
||||
|
||||
std::size_t getSize() const;
|
||||
const void* getDeviceMemRegion() const;
|
||||
void* getDeviceMemRegion();
|
||||
|
||||
void* getDevicePtr(const std::string& name);
|
||||
|
||||
bool checkHostPtr(const void* ptr) const;
|
||||
|
||||
private:
|
||||
std::size_t _size = 0;
|
||||
|
||||
std::unique_ptr<DeviceMem> _device;
|
||||
std::map<std::string, std::size_t> _offsets;
|
||||
|
||||
static const std::size_t alignment = STANDARD_PAGE_SIZE;
|
||||
};
|
||||
|
||||
} // namespace zeroMemory
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "zero_executor.hpp"
|
||||
#include "zero_memory.hpp"
|
||||
#include "zero_profiling.hpp"
|
||||
#include "zero_utils.hpp"
|
||||
#include "zero_wrappers.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
struct Pipeline {
|
||||
public:
|
||||
Pipeline() = default;
|
||||
Pipeline(const Pipeline&) = delete;
|
||||
Pipeline(Pipeline&&) = delete;
|
||||
Pipeline& operator=(const Pipeline&) = delete;
|
||||
Pipeline& operator=(Pipeline&&) = delete;
|
||||
virtual ~Pipeline() = default;
|
||||
|
||||
virtual void push() = 0;
|
||||
virtual void pull() = 0;
|
||||
virtual void reset() const = 0;
|
||||
|
||||
protected:
|
||||
zeroMemory::MemoryManagementUnit _deviceInputs;
|
||||
zeroMemory::MemoryManagementUnit _deviceOutputs;
|
||||
};
|
||||
|
||||
std::unique_ptr<Pipeline> makePipeline(const std::shared_ptr<const IExecutor>& executorPtr,
|
||||
const Config& config,
|
||||
zeroProfiling::ProfilingPool& profiling_pool,
|
||||
zeroProfiling::ProfilingQuery& profiling_query,
|
||||
std::shared_ptr<zeroProfiling::NpuInferProfiling> npu_profiling,
|
||||
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors);
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include <climits>
|
||||
#include <map>
|
||||
|
||||
#include "intel_npu/al/config/compiler.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "openvino/runtime/profiling_info.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
namespace zeroProfiling {
|
||||
|
||||
using LayerStatistics = std::vector<ov::ProfilingInfo>;
|
||||
|
||||
constexpr uint32_t POOL_SIZE = 1;
|
||||
|
||||
struct ProfilingPool {
|
||||
ProfilingPool(ze_graph_handle_t graph_handle,
|
||||
uint32_t profiling_count,
|
||||
ze_graph_profiling_dditable_ext_t* graph_profiling_ddi_table_ext)
|
||||
: _graph_handle(graph_handle),
|
||||
_profiling_count(profiling_count),
|
||||
_graph_profiling_ddi_table_ext(graph_profiling_ddi_table_ext) {}
|
||||
ProfilingPool(const ProfilingPool&) = delete;
|
||||
ProfilingPool& operator=(const ProfilingPool&) = delete;
|
||||
bool create();
|
||||
|
||||
~ProfilingPool();
|
||||
|
||||
ze_graph_handle_t _graph_handle;
|
||||
const uint32_t _profiling_count;
|
||||
ze_graph_profiling_pool_handle_t _handle = nullptr;
|
||||
ze_graph_profiling_dditable_ext_t* _graph_profiling_ddi_table_ext = nullptr;
|
||||
};
|
||||
|
||||
struct ProfilingQuery {
|
||||
ProfilingQuery(uint32_t index,
|
||||
ze_device_handle_t device_handle,
|
||||
ze_graph_profiling_dditable_ext_t* graph_profiling_ddi_table_ext)
|
||||
: _index(index),
|
||||
_device_handle(device_handle),
|
||||
_graph_profiling_ddi_table_ext(graph_profiling_ddi_table_ext) {}
|
||||
ProfilingQuery(const ProfilingQuery&) = delete;
|
||||
ProfilingQuery& operator=(const ProfilingQuery&) = delete;
|
||||
void create(const ze_graph_profiling_pool_handle_t& profiling_pool);
|
||||
ze_graph_profiling_query_handle_t getHandle() const {
|
||||
return _handle;
|
||||
}
|
||||
LayerStatistics getLayerStatistics() const;
|
||||
template <class ProfilingData>
|
||||
std::vector<ProfilingData> getData() const;
|
||||
~ProfilingQuery();
|
||||
|
||||
private:
|
||||
void queryGetData(const ze_graph_profiling_type_t profilingType, uint32_t* pSize, uint8_t* pData) const;
|
||||
void getProfilingProperties(ze_device_profiling_data_properties_t* properties) const;
|
||||
void verifyProfilingProperties() const;
|
||||
|
||||
const uint32_t _index;
|
||||
ze_device_handle_t _device_handle;
|
||||
ze_graph_profiling_query_handle_t _handle = nullptr;
|
||||
ze_graph_profiling_dditable_ext_t* _graph_profiling_ddi_table_ext = nullptr;
|
||||
};
|
||||
|
||||
extern template std::vector<uint8_t> ProfilingQuery::getData<uint8_t>() const;
|
||||
|
||||
using NpuInferStatistics = std::vector<ov::ProfilingInfo>;
|
||||
|
||||
struct NpuInferProfiling final {
|
||||
explicit NpuInferProfiling(ze_context_handle_t context, ze_device_handle_t device_handle, ov::log::Level loglevel);
|
||||
NpuInferProfiling(const NpuInferProfiling&) = delete;
|
||||
NpuInferProfiling& operator=(const NpuInferProfiling&) = delete;
|
||||
NpuInferProfiling(NpuInferProfiling&&) = delete;
|
||||
NpuInferProfiling& operator=(NpuInferProfiling&&) = delete;
|
||||
|
||||
void sampleNpuTimestamps();
|
||||
NpuInferStatistics getNpuInferStatistics() const;
|
||||
|
||||
~NpuInferProfiling();
|
||||
|
||||
/// Buffers allocated by ZE driver
|
||||
void* npu_ts_infer_start = 0;
|
||||
void* npu_ts_infer_end = 0;
|
||||
|
||||
private:
|
||||
ze_context_handle_t _context = nullptr;
|
||||
ze_device_handle_t _device_handle;
|
||||
ov::log::Level _loglevel;
|
||||
Logger _logger;
|
||||
ze_device_properties_t _dev_properties;
|
||||
int64_t _npu_infer_stats_min_cc = LLONG_MAX;
|
||||
int64_t _npu_infer_stats_max_cc = 0;
|
||||
int64_t _npu_infer_stats_accu_cc = 0;
|
||||
uint32_t _npu_infer_stats_cnt = 0;
|
||||
uint32_t _npu_infer_logidx = 0;
|
||||
static const uint32_t _npu_infer_log_maxsize = 1024;
|
||||
/// rolling buffer to store duration of last <_npu_infer_log_maxsize number> infers
|
||||
int64_t _npu_infer_duration_log[_npu_infer_log_maxsize];
|
||||
|
||||
/// Helper function to convert npu clockcycles to usec
|
||||
int64_t convertCCtoUS(int64_t val_cc) const;
|
||||
};
|
||||
|
||||
} // namespace zeroProfiling
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include "intel_npu/al/config/runtime.hpp"
|
||||
|
||||
/**
|
||||
* @brief Last version of Table of Graph Extension functions used within plugin
|
||||
*/
|
||||
using ze_graph_dditable_ext_last_t = ze_graph_dditable_ext_1_5_t;
|
||||
|
||||
/**
|
||||
* @brief Table of Graph Extension functions pointers and function wrappers
|
||||
* @details Use original Graph Extension functions pointers from driver for function from within lower driver versions.
|
||||
* Use function wrappers for function from within higher driver versions in order to throw when loaded driver is older
|
||||
* than required
|
||||
*/
|
||||
struct ze_graph_dditable_ext_decorator final {
|
||||
private:
|
||||
ze_graph_dditable_ext_last_t* const _impl;
|
||||
const uint32_t _driverExtVersion;
|
||||
|
||||
ze_graph_dditable_ext_decorator(const ze_graph_dditable_ext_decorator&) = delete;
|
||||
ze_graph_dditable_ext_decorator(ze_graph_dditable_ext_decorator&&) = delete;
|
||||
|
||||
void throwWhenUnsupported(const std::string func, uint32_t since) {
|
||||
if (_driverExtVersion < since) {
|
||||
OPENVINO_THROW("L0 extension function ",
|
||||
func,
|
||||
" is only available with driver version ",
|
||||
ZE_MAJOR_VERSION(since),
|
||||
".",
|
||||
ZE_MINOR_VERSION(since),
|
||||
" or later");
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
ze_graph_dditable_ext_decorator(ze_graph_dditable_ext_last_t* impl, uint32_t driverExtVersion)
|
||||
: _impl(impl),
|
||||
_driverExtVersion(driverExtVersion) {
|
||||
// version 1.0
|
||||
pfnCreate = _impl->pfnCreate;
|
||||
pfnDestroy = _impl->pfnDestroy;
|
||||
pfnGetProperties = _impl->pfnGetProperties;
|
||||
pfnGetArgumentProperties = _impl->pfnGetArgumentProperties;
|
||||
pfnSetArgumentValue = _impl->pfnSetArgumentValue;
|
||||
pfnAppendGraphInitialize = _impl->pfnAppendGraphInitialize;
|
||||
pfnAppendGraphExecute = _impl->pfnAppendGraphExecute;
|
||||
pfnGetNativeBinary = _impl->pfnGetNativeBinary;
|
||||
pfnDeviceGetGraphProperties = _impl->pfnDeviceGetGraphProperties;
|
||||
|
||||
// version 1.1
|
||||
pfnGraphGetArgumentMetadata = _impl->pfnGraphGetArgumentMetadata;
|
||||
pfnGetArgumentProperties2 = _impl->pfnGetArgumentProperties2;
|
||||
|
||||
// version 1.2
|
||||
pfnGetArgumentProperties3 = _impl->pfnGetArgumentProperties3;
|
||||
|
||||
// version 1.3
|
||||
pfnQueryNetworkCreate = _impl->pfnQueryNetworkCreate;
|
||||
pfnQueryNetworkDestroy = _impl->pfnQueryNetworkDestroy;
|
||||
pfnQueryNetworkGetSupportedLayers = _impl->pfnQueryNetworkGetSupportedLayers;
|
||||
|
||||
// version 1.4
|
||||
pfnBuildLogGetString = _impl->pfnBuildLogGetString;
|
||||
|
||||
// version 1.5
|
||||
// wrappers replace pointers
|
||||
}
|
||||
~ze_graph_dditable_ext_decorator() = default;
|
||||
|
||||
// version 1.0
|
||||
ze_pfnGraphCreate_ext_t pfnCreate;
|
||||
ze_pfnGraphDestroy_ext_t pfnDestroy;
|
||||
ze_pfnGraphGetProperties_ext_t pfnGetProperties;
|
||||
ze_pfnGraphGetArgumentProperties_ext_t pfnGetArgumentProperties;
|
||||
ze_pfnGraphSetArgumentValue_ext_t pfnSetArgumentValue;
|
||||
ze_pfnAppendGraphInitialize_ext_t pfnAppendGraphInitialize;
|
||||
ze_pfnAppendGraphExecute_ext_t pfnAppendGraphExecute;
|
||||
ze_pfnGraphGetNativeBinary_ext_t pfnGetNativeBinary;
|
||||
ze_pfnDeviceGetGraphProperties_ext_t pfnDeviceGetGraphProperties;
|
||||
|
||||
// version 1.1
|
||||
ze_pfnGraphGetArgumentMetadata_ext_t pfnGraphGetArgumentMetadata;
|
||||
ze_pfnGraphGetArgumentProperties_ext_2_t pfnGetArgumentProperties2;
|
||||
|
||||
// version 1.2
|
||||
ze_pfnGraphGetArgumentProperties_ext_3_t pfnGetArgumentProperties3;
|
||||
|
||||
// version 1.3
|
||||
ze_pfnGraphQueryNetworkCreate_ext_t pfnQueryNetworkCreate;
|
||||
ze_pfnGraphQueryNetworkDestroy_ext_t pfnQueryNetworkDestroy;
|
||||
ze_pfnGraphQueryNetworkGetSupportedLayers_ext_t pfnQueryNetworkGetSupportedLayers;
|
||||
|
||||
// version 1.4
|
||||
ze_pfnGraphBuildLogGetString_ext_t pfnBuildLogGetString;
|
||||
|
||||
// version 1.5
|
||||
ze_result_t ZE_APICALL pfnCreate2(ze_context_handle_t hContext,
|
||||
ze_device_handle_t hDevice,
|
||||
const ze_graph_desc_2_t* desc,
|
||||
ze_graph_handle_t* phGraph) {
|
||||
throwWhenUnsupported("pfnCreate2", ZE_GRAPH_EXT_VERSION_1_5);
|
||||
return _impl->pfnCreate2(hContext, hDevice, desc, phGraph);
|
||||
}
|
||||
|
||||
ze_result_t ZE_APICALL pfnQueryNetworkCreate2(ze_context_handle_t hContext,
|
||||
ze_device_handle_t hDevice,
|
||||
const ze_graph_desc_2_t* desc,
|
||||
ze_graph_query_network_handle_t* phGraphQueryNetwork) {
|
||||
throwWhenUnsupported("pfnQueryNetworkCreate2", ZE_GRAPH_EXT_VERSION_1_5);
|
||||
return _impl->pfnQueryNetworkCreate2(hContext, hDevice, desc, phGraphQueryNetwork);
|
||||
}
|
||||
|
||||
ze_result_t ZE_APICALL pfnQueryContextMemory(ze_context_handle_t hContext,
|
||||
ze_graph_memory_query_type_t type,
|
||||
ze_graph_memory_query_t* query) {
|
||||
throwWhenUnsupported("pfnQueryContextMemory", ZE_GRAPH_EXT_VERSION_1_5);
|
||||
return _impl->pfnQueryContextMemory(hContext, type, query);
|
||||
}
|
||||
};
|
||||
|
||||
using ze_graph_dditable_ext_curr_t = ze_graph_dditable_ext_decorator;
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <limits.h>
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include "intel_npu/al/config/runtime.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "intel_npu/utils/zero/zero_result.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
namespace zeroUtils {
|
||||
|
||||
static inline void throwOnFail(const std::string& step, const ze_result_t result) {
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("L0 ",
|
||||
step,
|
||||
" result: ",
|
||||
ze_result_to_string(result),
|
||||
", code 0x",
|
||||
std::hex,
|
||||
uint64_t(result),
|
||||
" - ",
|
||||
ze_result_to_description(result));
|
||||
}
|
||||
}
|
||||
|
||||
static inline void throwOnFail(const std::string& step, const ze_result_t result, const std::string& hintOnError) {
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("L0 ",
|
||||
step,
|
||||
" result: ",
|
||||
ze_result_to_string(result),
|
||||
", code 0x",
|
||||
std::hex,
|
||||
uint64_t(result),
|
||||
" - ",
|
||||
ze_result_to_description(result),
|
||||
". ",
|
||||
hintOnError);
|
||||
}
|
||||
}
|
||||
|
||||
static inline ze_command_queue_priority_t toZeQueuePriority(const ov::hint::Priority& val) {
|
||||
switch (val) {
|
||||
case ov::hint::Priority::LOW:
|
||||
return ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_LOW;
|
||||
case ov::hint::Priority::MEDIUM:
|
||||
return ZE_COMMAND_QUEUE_PRIORITY_NORMAL;
|
||||
case ov::hint::Priority::HIGH:
|
||||
return ZE_COMMAND_QUEUE_PRIORITY_PRIORITY_HIGH;
|
||||
default:
|
||||
OPENVINO_THROW("Incorrect queue priority.");
|
||||
}
|
||||
}
|
||||
|
||||
static inline std::size_t precisionToSize(const ze_graph_argument_precision_t val) {
|
||||
switch (val) {
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_INT4:
|
||||
return 4;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_UINT4:
|
||||
return 4;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_INT8:
|
||||
return 8;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_UINT8:
|
||||
return 8;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_INT16:
|
||||
return 16;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_UINT16:
|
||||
return 16;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_INT32:
|
||||
return 32;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_UINT32:
|
||||
return 32;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_INT64:
|
||||
return 64;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_UINT64:
|
||||
return 64;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_BF16:
|
||||
return 16;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_FP16:
|
||||
return 16;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_FP32:
|
||||
return 32;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_FP64:
|
||||
return 64;
|
||||
case ZE_GRAPH_ARGUMENT_PRECISION_BIN:
|
||||
return 1;
|
||||
default:
|
||||
OPENVINO_THROW("precisionToSize switch->default reached");
|
||||
}
|
||||
}
|
||||
|
||||
static inline ze_graph_argument_precision_t getZePrecision(const ov::element::Type_t precision) {
|
||||
switch (precision) {
|
||||
case ov::element::Type_t::i4:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_INT4;
|
||||
case ov::element::Type_t::u4:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_UINT4;
|
||||
case ov::element::Type_t::i8:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_INT8;
|
||||
case ov::element::Type_t::u8:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_UINT8;
|
||||
case ov::element::Type_t::i16:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_INT16;
|
||||
case ov::element::Type_t::u16:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_UINT16;
|
||||
case ov::element::Type_t::i32:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_INT32;
|
||||
case ov::element::Type_t::u32:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_UINT32;
|
||||
case ov::element::Type_t::i64:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_INT64;
|
||||
case ov::element::Type_t::u64:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_UINT64;
|
||||
case ov::element::Type_t::bf16:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_BF16;
|
||||
case ov::element::Type_t::f16:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_FP16;
|
||||
case ov::element::Type_t::f32:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_FP32;
|
||||
case ov::element::Type_t::f64:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_FP64;
|
||||
case ov::element::Type_t::u1:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_BIN;
|
||||
default:
|
||||
return ZE_GRAPH_ARGUMENT_PRECISION_UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
static inline std::size_t layoutCount(const ze_graph_argument_layout_t val) {
|
||||
switch (val) {
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_NCHW:
|
||||
return 4;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_NHWC:
|
||||
return 4;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_NCDHW:
|
||||
return 5;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_NDHWC:
|
||||
return 5;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_OIHW:
|
||||
return 4;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_C:
|
||||
return 1;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_CHW:
|
||||
return 3;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_HW:
|
||||
return 2;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_NC:
|
||||
return 2;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_CN:
|
||||
return 2;
|
||||
case ZE_GRAPH_ARGUMENT_LAYOUT_ANY:
|
||||
// When input has empty shape, val is ZE_GRAPH_ARGUMENT_LAYOUT_ANY
|
||||
// Add this to pass Single Layer Test on Windows
|
||||
return 0;
|
||||
default:
|
||||
OPENVINO_THROW("layoutCount switch->default reached");
|
||||
}
|
||||
}
|
||||
|
||||
static inline std::size_t getSizeIOBytes(const ze_graph_argument_properties_t& argument) {
|
||||
std::size_t num_elements = 1;
|
||||
for (std::size_t i = 0; i < layoutCount(argument.deviceLayout); ++i) {
|
||||
num_elements *= argument.dims[i];
|
||||
}
|
||||
const std::size_t size_in_bits = num_elements * precisionToSize(argument.devicePrecision);
|
||||
const std::size_t size_in_bytes = (size_in_bits + (CHAR_BIT - 1)) / CHAR_BIT;
|
||||
return size_in_bytes;
|
||||
}
|
||||
|
||||
static inline uint32_t findGroupOrdinal(
|
||||
const std::vector<ze_command_queue_group_properties_t>& command_group_properties,
|
||||
const ze_device_properties_t& properties) {
|
||||
auto log = Logger::global().clone("findGroupOrdinal");
|
||||
|
||||
if (properties.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) {
|
||||
for (uint32_t index = 0; index < command_group_properties.size(); ++index) {
|
||||
const auto& flags = command_group_properties[index].flags;
|
||||
if ((flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) != 0 &&
|
||||
(flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) == 0) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
// if we don't find a group where only the proper flag is enabled then search for a group where that flag is
|
||||
// enabled
|
||||
for (uint32_t index = 0; index < command_group_properties.size(); ++index) {
|
||||
const auto& flags = command_group_properties[index].flags;
|
||||
if (flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
// if still don't find compute flag, return a warning
|
||||
log.warning("Fail to find a command queue group that contains compute flag, it will be set to 0.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (uint32_t index = 0; index < command_group_properties.size(); ++index) {
|
||||
const auto& flags = command_group_properties[index].flags;
|
||||
if ((flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) != 0 &&
|
||||
(flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) != 0) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
// if still don't find compute and copy flag, return a warning
|
||||
log.warning("Fail to find a command queue group that contains compute and copy flags, it will be set to 0.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace zeroUtils
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "zero_types.hpp"
|
||||
#include "zero_utils.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
class CommandList;
|
||||
class CommandQueue;
|
||||
|
||||
enum stage {
|
||||
UPLOAD,
|
||||
EXECUTE,
|
||||
READBACK,
|
||||
|
||||
COUNT
|
||||
};
|
||||
|
||||
class EventPool {
|
||||
public:
|
||||
EventPool() = delete;
|
||||
EventPool(ze_device_handle_t device_handle,
|
||||
const ze_context_handle_t& context,
|
||||
uint32_t event_count,
|
||||
const Config& config);
|
||||
EventPool(const EventPool&) = delete;
|
||||
EventPool(EventPool&&) = delete;
|
||||
EventPool& operator=(const EventPool&) = delete;
|
||||
EventPool& operator=(EventPool&&) = delete;
|
||||
~EventPool();
|
||||
inline ze_event_pool_handle_t handle() const {
|
||||
return _handle;
|
||||
}
|
||||
|
||||
private:
|
||||
ze_event_pool_handle_t _handle = nullptr;
|
||||
|
||||
Logger _log;
|
||||
};
|
||||
|
||||
class Event {
|
||||
public:
|
||||
Event() = delete;
|
||||
Event(const ze_event_pool_handle_t& event_pool, uint32_t event_index, const Config& config);
|
||||
Event(const Event&) = delete;
|
||||
Event(Event&&) = delete;
|
||||
Event& operator=(const Event&) = delete;
|
||||
Event& operator=(Event&&) = delete;
|
||||
|
||||
void AppendSignalEvent(CommandList& command_list) const;
|
||||
void AppendWaitOnEvent(CommandList& command_list);
|
||||
void AppendEventReset(CommandList& command_list) const;
|
||||
void hostSynchronize() const;
|
||||
void reset() const;
|
||||
~Event();
|
||||
|
||||
private:
|
||||
ze_event_handle_t _handle = nullptr;
|
||||
|
||||
Logger _log;
|
||||
};
|
||||
|
||||
class CommandList {
|
||||
public:
|
||||
friend class CommandQueue;
|
||||
CommandList() = delete;
|
||||
CommandList(const ze_device_handle_t& device_handle,
|
||||
const ze_context_handle_t& context,
|
||||
ze_graph_dditable_ext_curr_t* graph_ddi_table_ext,
|
||||
const Config& config,
|
||||
const uint32_t& group_ordinal);
|
||||
CommandList(const CommandList&) = delete;
|
||||
CommandList(CommandList&&) = delete;
|
||||
CommandList& operator=(const CommandList&) = delete;
|
||||
CommandList& operator=(CommandList&&) = delete;
|
||||
|
||||
void reset() const;
|
||||
void appendMemoryCopy(void* dst, const void* src, const std::size_t size) const;
|
||||
void appendGraphInitialize(const ze_graph_handle_t& graph_handle) const;
|
||||
void appendGraphExecute(const ze_graph_handle_t& graph_handle,
|
||||
const ze_graph_profiling_query_handle_t& profiling_query_handle) const;
|
||||
void appendNpuTimestamp(uint64_t* timestamp_buff) const;
|
||||
void appendBarrier() const;
|
||||
void close() const;
|
||||
~CommandList();
|
||||
|
||||
inline ze_command_list_handle_t handle() const {
|
||||
return _handle;
|
||||
}
|
||||
|
||||
private:
|
||||
ze_command_list_handle_t _handle = nullptr;
|
||||
const ze_context_handle_t _context = nullptr;
|
||||
ze_graph_dditable_ext_curr_t* _graph_ddi_table_ext = nullptr;
|
||||
|
||||
Logger _log;
|
||||
};
|
||||
|
||||
class Fence {
|
||||
public:
|
||||
Fence() = delete;
|
||||
Fence(const CommandQueue& command_queue, const Config& config);
|
||||
Fence(const Fence&) = delete;
|
||||
Fence(Fence&&) = delete;
|
||||
Fence& operator=(const Fence&) = delete;
|
||||
Fence& operator=(Fence&&) = delete;
|
||||
|
||||
void reset() const;
|
||||
void hostSynchronize() const;
|
||||
~Fence();
|
||||
inline ze_fence_handle_t handle() const {
|
||||
return _handle;
|
||||
}
|
||||
|
||||
private:
|
||||
ze_fence_handle_t _handle = nullptr;
|
||||
|
||||
Logger _log;
|
||||
};
|
||||
|
||||
class CommandQueue {
|
||||
public:
|
||||
CommandQueue() = delete;
|
||||
CommandQueue(const ze_device_handle_t& device_handle,
|
||||
const ze_context_handle_t& context,
|
||||
const ze_command_queue_priority_t& priority,
|
||||
const Config& config,
|
||||
const uint32_t& group_ordinal);
|
||||
CommandQueue(const CommandQueue&) = delete;
|
||||
CommandQueue(CommandQueue&&) = delete;
|
||||
CommandQueue& operator=(const CommandQueue&) = delete;
|
||||
CommandQueue& operator=(CommandQueue&&) = delete;
|
||||
|
||||
void executeCommandList(CommandList& command_list) const;
|
||||
void executeCommandList(CommandList& command_list, Fence& fence) const;
|
||||
~CommandQueue();
|
||||
inline ze_command_queue_handle_t handle() const {
|
||||
return _handle;
|
||||
}
|
||||
|
||||
private:
|
||||
ze_command_queue_handle_t _handle = nullptr;
|
||||
ze_context_handle_t _context = nullptr;
|
||||
|
||||
Logger _log;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "zero_backend.hpp"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
#include "zero_device.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
ZeroEngineBackend::ZeroEngineBackend(const Config& config) {
|
||||
Logger::global().setLevel(config.get<LOG_LEVEL>());
|
||||
|
||||
_instance = std::make_shared<ZeroInitStructsHolder>();
|
||||
|
||||
auto device = std::make_shared<ZeroDevice>(_instance);
|
||||
_devices.emplace(std::make_pair(device->getName(), device));
|
||||
}
|
||||
|
||||
ZeroEngineBackend::~ZeroEngineBackend() = default;
|
||||
|
||||
const std::shared_ptr<IDevice> ZeroEngineBackend::getDevice() const {
|
||||
if (_devices.empty()) {
|
||||
return {};
|
||||
} else {
|
||||
return _devices.begin()->second;
|
||||
}
|
||||
}
|
||||
|
||||
const std::shared_ptr<IDevice> ZeroEngineBackend::getDevice(const std::string& /*name*/) const {
|
||||
// TODO Add the search of the device by platform & slice
|
||||
return getDevice();
|
||||
}
|
||||
|
||||
const std::vector<std::string> ZeroEngineBackend::getDeviceNames() const {
|
||||
std::vector<std::string> devicesNames;
|
||||
std::for_each(_devices.cbegin(), _devices.cend(), [&devicesNames](const auto& device) {
|
||||
devicesNames.push_back(device.first);
|
||||
});
|
||||
return devicesNames;
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "zero_device.hpp"
|
||||
|
||||
#include <ze_api.h>
|
||||
|
||||
#include "intel_npu/al/itt.hpp"
|
||||
#include "zero_executor.hpp"
|
||||
#include "zero_infer_request.hpp"
|
||||
|
||||
using namespace intel_npu;
|
||||
|
||||
ZeroDevice::ZeroDevice(const std::shared_ptr<ZeroInitStructsHolder>& initStructs)
|
||||
: _initStructs(initStructs),
|
||||
_graph_ddi_table_ext(_initStructs->getGraphDdiTable()),
|
||||
log("ZeroDevice", Logger::global().level()) {
|
||||
device_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
|
||||
zeroUtils::throwOnFail("zeDeviceGetProperties",
|
||||
zeDeviceGetProperties(_initStructs->getDevice(), &device_properties));
|
||||
driver_properties.stype = ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES;
|
||||
zeroUtils::throwOnFail("zeDriverGetProperties",
|
||||
zeDriverGetProperties(_initStructs->getDriver(), &driver_properties));
|
||||
|
||||
std::vector<ze_command_queue_group_properties_t> command_group_properties;
|
||||
uint32_t command_queue_group_count = 0;
|
||||
// Discover all command queue groups
|
||||
zeroUtils::throwOnFail(
|
||||
"zeDeviceGetCommandQueueGroupProperties",
|
||||
zeDeviceGetCommandQueueGroupProperties(_initStructs->getDevice(), &command_queue_group_count, nullptr));
|
||||
|
||||
command_group_properties.resize(command_queue_group_count);
|
||||
|
||||
for (auto& prop : command_group_properties) {
|
||||
prop.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES;
|
||||
prop.pNext = nullptr;
|
||||
}
|
||||
|
||||
zeroUtils::throwOnFail("zeDeviceGetCommandQueueGroupProperties",
|
||||
zeDeviceGetCommandQueueGroupProperties(_initStructs->getDevice(),
|
||||
&command_queue_group_count,
|
||||
command_group_properties.data()));
|
||||
|
||||
// Find the corresponding command queue group.
|
||||
_group_ordinal = zeroUtils::findGroupOrdinal(command_group_properties, device_properties);
|
||||
}
|
||||
|
||||
std::shared_ptr<IExecutor> ZeroDevice::createExecutor(
|
||||
const std::shared_ptr<const NetworkDescription>& networkDescription,
|
||||
const Config& config) {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Device::createExecutor");
|
||||
return std::make_shared<ZeroExecutor>(_initStructs, networkDescription, config, _group_ordinal);
|
||||
}
|
||||
|
||||
std::string ZeroDevice::getName() const {
|
||||
// KMD is setting usDeviceID from VpuFamilyID.h
|
||||
#define NPU_3700_DEVICE_ID 0x6240
|
||||
#define NPU_3720_P_DEVICE_ID 0x7D1D
|
||||
#define NPU_3720_S_DEVICE_ID 0xAD1D
|
||||
|
||||
std::string name;
|
||||
switch (device_properties.deviceId) {
|
||||
case NPU_3700_DEVICE_ID:
|
||||
name = "3700";
|
||||
break;
|
||||
case NPU_3720_P_DEVICE_ID:
|
||||
case NPU_3720_S_DEVICE_ID:
|
||||
name = ov::intel_npu::Platform::NPU3720;
|
||||
break;
|
||||
default:
|
||||
name = "AUTO_DETECT";
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
std::string ZeroDevice::getFullDeviceName() const {
|
||||
return device_properties.name;
|
||||
}
|
||||
|
||||
IDevice::Uuid ZeroDevice::getUuid() const {
|
||||
Uuid uuid{};
|
||||
static_assert(sizeof(device_properties.uuid.id) == uuid.uuid.size(),
|
||||
"ze_device_uuid_t::id size doesn't match intel_npu::Uuid::uuid size");
|
||||
|
||||
std::copy(std::begin(device_properties.uuid.id), std::end(device_properties.uuid.id), std::begin(uuid.uuid));
|
||||
|
||||
return uuid;
|
||||
}
|
||||
|
||||
uint32_t ZeroDevice::getDriverVersion() const {
|
||||
return driver_properties.driverVersion;
|
||||
}
|
||||
|
||||
uint32_t ZeroDevice::getSubDevId() const {
|
||||
return device_properties.subdeviceId;
|
||||
}
|
||||
|
||||
uint32_t ZeroDevice::getMaxNumSlices() const {
|
||||
return device_properties.numSlices;
|
||||
}
|
||||
|
||||
uint64_t ZeroDevice::getAllocMemSize() const {
|
||||
ze_graph_memory_query_t query{};
|
||||
zeroUtils::throwOnFail(
|
||||
"pfnQueryContextMemory",
|
||||
_graph_ddi_table_ext->pfnQueryContextMemory(_initStructs->getContext(), ZE_GRAPH_QUERY_MEMORY_DDR, &query));
|
||||
return query.allocated;
|
||||
}
|
||||
|
||||
uint64_t ZeroDevice::getTotalMemSize() const {
|
||||
ze_graph_memory_query_t query{};
|
||||
zeroUtils::throwOnFail(
|
||||
"pfnQueryContextMemory",
|
||||
_graph_ddi_table_ext->pfnQueryContextMemory(_initStructs->getContext(), ZE_GRAPH_QUERY_MEMORY_DDR, &query));
|
||||
return query.total;
|
||||
}
|
||||
|
||||
std::shared_ptr<SyncInferRequest> ZeroDevice::createInferRequest(
|
||||
const std::shared_ptr<const ICompiledModel>& compiledModel,
|
||||
const std::shared_ptr<IExecutor>& executor,
|
||||
const Config& config) {
|
||||
return std::make_shared<ZeroInferRequest>(_initStructs, compiledModel, executor, config);
|
||||
}
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "zero_executor.hpp"
|
||||
|
||||
#include <ze_api.h>
|
||||
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
#include "intel_npu/al/itt.hpp"
|
||||
#include "zero_device.hpp"
|
||||
#include "zero_utils.hpp"
|
||||
|
||||
using namespace intel_npu;
|
||||
|
||||
ZeroExecutor::ZeroExecutor(const std::shared_ptr<const ZeroInitStructsHolder>& initStructs,
|
||||
const std::shared_ptr<const NetworkDescription>& networkDescription,
|
||||
const Config& config,
|
||||
const uint32_t& group_ordinal)
|
||||
: _config(config),
|
||||
_logger("Graph", _config.get<LOG_LEVEL>()),
|
||||
_initStructs(initStructs),
|
||||
_networkDesc(networkDescription),
|
||||
_graph_ddi_table_ext(_initStructs->getGraphDdiTable()),
|
||||
_group_ordinal(group_ordinal),
|
||||
_command_queues{{std::make_shared<CommandQueue>(_initStructs->getDevice(),
|
||||
_initStructs->getContext(),
|
||||
zeroUtils::toZeQueuePriority(_config.get<MODEL_PRIORITY>()),
|
||||
_config,
|
||||
group_ordinal),
|
||||
std::make_shared<CommandQueue>(_initStructs->getDevice(),
|
||||
_initStructs->getContext(),
|
||||
zeroUtils::toZeQueuePriority(_config.get<MODEL_PRIORITY>()),
|
||||
_config,
|
||||
group_ordinal),
|
||||
std::make_shared<CommandQueue>(_initStructs->getDevice(),
|
||||
_initStructs->getContext(),
|
||||
zeroUtils::toZeQueuePriority(_config.get<MODEL_PRIORITY>()),
|
||||
_config,
|
||||
group_ordinal)}} {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Executor::ZeroExecutor");
|
||||
CommandList graph_command_list(_initStructs->getDevice(),
|
||||
_initStructs->getContext(),
|
||||
_initStructs->getGraphDdiTable(),
|
||||
_config,
|
||||
_group_ordinal);
|
||||
CommandQueue graph_command_queue(_initStructs->getDevice(),
|
||||
_initStructs->getContext(),
|
||||
ZE_COMMAND_QUEUE_PRIORITY_NORMAL,
|
||||
_config,
|
||||
_group_ordinal);
|
||||
Fence fence(graph_command_queue, _config);
|
||||
ze_device_properties_t properties = {};
|
||||
properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
|
||||
zeroUtils::throwOnFail("zeDeviceGetProperties", zeDeviceGetProperties(_initStructs->getDevice(), &properties));
|
||||
|
||||
OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_GRAPH, itt::domains::LevelZeroBackend, "Executor::ZeroExecutor", "graphCreate");
|
||||
ze_graph_desc_t desc{ZE_STRUCTURE_TYPE_GRAPH_DESC_PROPERTIES,
|
||||
nullptr,
|
||||
ZE_GRAPH_FORMAT_NATIVE,
|
||||
_networkDesc->compiledNetwork.size(),
|
||||
_networkDesc->compiledNetwork.data(),
|
||||
nullptr};
|
||||
zeroUtils::throwOnFail(
|
||||
"pfnCreate",
|
||||
_graph_ddi_table_ext->pfnCreate(_initStructs->getContext(), _initStructs->getDevice(), &desc, &_graph));
|
||||
|
||||
OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGetProperties");
|
||||
zeroUtils::throwOnFail("pfnGetProperties", _graph_ddi_table_ext->pfnGetProperties(_graph, &_props));
|
||||
|
||||
OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGetArgumentProperties");
|
||||
for (uint32_t index = 0; index < _props.numGraphArgs; ++index) {
|
||||
ze_graph_argument_properties_t arg;
|
||||
zeroUtils::throwOnFail("pfnGetArgumentProperties",
|
||||
_graph_ddi_table_ext->pfnGetArgumentProperties(_graph, index, &arg));
|
||||
if (ZE_GRAPH_ARGUMENT_TYPE_INPUT == arg.type) {
|
||||
_inputs_desc_map.emplace(std::make_pair(std::string(arg.name), ArgumentDescriptor{arg, index}));
|
||||
} else {
|
||||
_outputs_desc_map.emplace(std::make_pair(std::string(arg.name), ArgumentDescriptor{arg, index}));
|
||||
}
|
||||
}
|
||||
OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "appendGraphInitialize");
|
||||
graph_command_list.appendGraphInitialize(_graph);
|
||||
graph_command_list.close();
|
||||
|
||||
OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "queue_execute");
|
||||
graph_command_queue.executeCommandList(graph_command_list, fence);
|
||||
fence.hostSynchronize();
|
||||
}
|
||||
|
||||
void ZeroExecutor::setArgumentValue(uint32_t argi_, const void* argv_) const {
|
||||
zeroUtils::throwOnFail("zeGraphSetArgumentValue", _graph_ddi_table_ext->pfnSetArgumentValue(_graph, argi_, argv_));
|
||||
}
|
||||
|
||||
ZeroExecutor::~ZeroExecutor() {
|
||||
auto result = _graph_ddi_table_ext->pfnDestroy(_graph);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
_logger.error("_graph_ddi_table_ext->pfnDestroy failed %#X", uint64_t(result));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,264 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "zero_infer_request.hpp"
|
||||
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
#include "intel_npu/al/config/compiler.hpp"
|
||||
#include "intel_npu/al/config/runtime.hpp"
|
||||
#include "intel_npu/al/itt.hpp"
|
||||
#include "intel_npu/al/prefix.hpp"
|
||||
#include "zero_memory.hpp"
|
||||
|
||||
using namespace intel_npu;
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr bool STATE_TENSOR = true;
|
||||
constexpr bool NOT_STATE_TENSOR = false;
|
||||
|
||||
/**
|
||||
* @brief Checks that the metadata of the provided descriptor corresponds to the values registered in the Level Zero
|
||||
* structure.
|
||||
* @param nodeDescriptor The OpenVINO API specific I/O descriptor which shall be compared.
|
||||
* @param zeDescriptor The Level Zero specific structure used for comparison.
|
||||
* @param name Tensor identifier used for error logging.
|
||||
*/
|
||||
void check_level_zero_attributes_match(const IONodeDescriptor& nodeDescriptor,
|
||||
const ZeroExecutor::ArgumentDescriptor& zeDescriptor,
|
||||
const std::string& name) {
|
||||
const ov::element::Type_t ovPrecision = nodeDescriptor.precision;
|
||||
const ze_graph_argument_precision_t zePrecision = zeDescriptor.info.devicePrecision;
|
||||
|
||||
if (zeroUtils::getZePrecision(ovPrecision) != zePrecision) {
|
||||
OPENVINO_THROW("Precision mismatch for parameter " + name);
|
||||
}
|
||||
|
||||
const std::vector<size_t>& ovDimensions = nodeDescriptor.originalShape.get_shape();
|
||||
|
||||
if (ovDimensions.size() > ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE) {
|
||||
OPENVINO_THROW(
|
||||
"Maximum number of dimensions supported: " + std::to_string(ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE) + '\n' +
|
||||
"Given: " + std::to_string(ovDimensions.size()));
|
||||
}
|
||||
|
||||
for (size_t index = 0; index < ovDimensions.size(); ++index) {
|
||||
if (ovDimensions[index] != zeDescriptor.info.dims[index]) {
|
||||
OPENVINO_THROW("Shape mismatch for parameter " + name);
|
||||
}
|
||||
}
|
||||
for (size_t index = ovDimensions.size(); index < ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE; ++index) {
|
||||
if (zeDescriptor.info.dims[index] != 0 && zeDescriptor.info.dims[index] != 1) {
|
||||
OPENVINO_THROW("Shape mismatch for parameter " + name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
ZeroInferRequest::ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>& backendPtr,
|
||||
const std::shared_ptr<const ICompiledModel>& compiledModel,
|
||||
const std::shared_ptr<const IExecutor>& executor,
|
||||
const Config& config)
|
||||
: SyncInferRequest(compiledModel),
|
||||
_executorPtr(executor),
|
||||
_executor(static_cast<const ZeroExecutor*>(_executorPtr.get())),
|
||||
_config(config),
|
||||
_logger("ZeroInferRequest", config.get<LOG_LEVEL>()),
|
||||
_profiling_pool(_executor->graph(),
|
||||
zeroProfiling::POOL_SIZE,
|
||||
_executor->getInitStructs()->getProfilingDdiTable()),
|
||||
_profiling_query(0,
|
||||
_executor->getInitStructs()->getDevice(),
|
||||
_executor->getInitStructs()->getProfilingDdiTable()) {
|
||||
const std::unordered_map<std::string, ZeroExecutor::ArgumentDescriptor>& executorInputDescriptors =
|
||||
_executor->inputs_desc_map();
|
||||
const std::unordered_map<std::string, ZeroExecutor::ArgumentDescriptor>& executorOutputDescriptors =
|
||||
_executor->outputs_desc_map();
|
||||
|
||||
auto proftype = config.get<PROFILING_TYPE>();
|
||||
if (proftype == ov::intel_npu::ProfilingType::INFER) {
|
||||
_npu_profiling = std::make_shared<zeroProfiling::NpuInferProfiling>(_executor->getInitStructs()->getContext(),
|
||||
_executor->getInitStructs()->getDevice(),
|
||||
_config.get<LOG_LEVEL>());
|
||||
}
|
||||
|
||||
ze_device_properties_t properties = {};
|
||||
properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
|
||||
zeroUtils::throwOnFail("zeDeviceGetProperties",
|
||||
zeDeviceGetProperties(_executor->getInitStructs()->getDevice(), &properties));
|
||||
|
||||
for (const std::string& inputName : _metadata.inputNames) {
|
||||
if (!executorInputDescriptors.count(inputName)) {
|
||||
OPENVINO_THROW("Invalid graph input descriptor key: " + inputName);
|
||||
}
|
||||
|
||||
const IONodeDescriptor& parameterDescriptor = _metadata.parameters.at(inputName);
|
||||
check_level_zero_attributes_match(parameterDescriptor, executorInputDescriptors.at(inputName), inputName);
|
||||
|
||||
ov::Allocator allocator;
|
||||
if (properties.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) {
|
||||
allocator = zeroMemory::HostMemAllocator(backendPtr, ZE_HOST_MEM_ALLOC_FLAG_BIAS_WRITE_COMBINED);
|
||||
} else {
|
||||
allocator = zeroMemory::HostMemAllocator(backendPtr);
|
||||
}
|
||||
|
||||
// The I/O buffers already allocated using the Level Zero API are being reused here
|
||||
allocate_tensor(inputName, parameterDescriptor, NOT_STATE_TENSOR, allocator);
|
||||
}
|
||||
|
||||
for (const std::string& outputName : _metadata.outputNames) {
|
||||
if (!executorOutputDescriptors.count(outputName)) {
|
||||
OPENVINO_THROW("Invalid graph output descriptor key: " + outputName);
|
||||
}
|
||||
|
||||
const IONodeDescriptor& resultDescriptor = _metadata.results.at(outputName);
|
||||
check_level_zero_attributes_match(resultDescriptor, executorOutputDescriptors.at(outputName), outputName);
|
||||
|
||||
auto allocator = zeroMemory::HostMemAllocator(backendPtr);
|
||||
|
||||
allocate_tensor(outputName, resultDescriptor, NOT_STATE_TENSOR, allocator);
|
||||
}
|
||||
|
||||
for (const std::string& stateName : _metadata.stateNames) {
|
||||
const std::string& stateInputBufferName = READVALUE_PREFIX + stateName;
|
||||
const std::string& stateOutputBufferName = ASSIGN_PREFIX + stateName;
|
||||
|
||||
if (!executorInputDescriptors.count(stateInputBufferName)) {
|
||||
OPENVINO_THROW("Invalid graph input descriptor key: " + stateInputBufferName);
|
||||
}
|
||||
if (!executorOutputDescriptors.count(stateOutputBufferName)) {
|
||||
OPENVINO_THROW("Invalid graph output descriptor key: " + stateOutputBufferName);
|
||||
}
|
||||
|
||||
const IONodeDescriptor& stateDescriptor = _metadata.states.at(stateName);
|
||||
check_level_zero_attributes_match(stateDescriptor,
|
||||
executorInputDescriptors.at(stateInputBufferName),
|
||||
stateInputBufferName);
|
||||
check_level_zero_attributes_match(stateDescriptor,
|
||||
executorOutputDescriptors.at(stateOutputBufferName),
|
||||
stateOutputBufferName);
|
||||
|
||||
auto allocator = zeroMemory::HostMemAllocator(backendPtr);
|
||||
|
||||
// Only one buffer per state variable is required, we'll use the "output" one since this one captures the latest
|
||||
// tensor value
|
||||
allocate_tensor(stateName, stateDescriptor, STATE_TENSOR, allocator);
|
||||
}
|
||||
|
||||
/// Construct pipepline
|
||||
_pipeline = makePipeline(_executorPtr, _config, _profiling_pool, _profiling_query, _npu_profiling, _copyAllTensors);
|
||||
}
|
||||
|
||||
void ZeroInferRequest::infer() {
|
||||
infer_async();
|
||||
get_result();
|
||||
}
|
||||
|
||||
void ZeroInferRequest::infer_async() {
|
||||
_logger.debug("InferRequest::infer_async started");
|
||||
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "infer_async");
|
||||
|
||||
for (const auto& name : _inputAndStateInputNames) {
|
||||
const std::shared_ptr<ov::ITensor>& inputTensor = _allTensors.at(name);
|
||||
const std::shared_ptr<ov::ITensor>& wrapperInputTensor = _copyAllTensors.at(name);
|
||||
|
||||
const uint8_t* tensorBuffer = reinterpret_cast<uint8_t*>(inputTensor->data());
|
||||
uint8_t* copyTensorBuffer = reinterpret_cast<uint8_t*>(wrapperInputTensor->data());
|
||||
|
||||
if (tensorBuffer != copyTensorBuffer) {
|
||||
if (tensorBuffer == nullptr || copyTensorBuffer == nullptr) {
|
||||
OPENVINO_THROW("Empty buffer");
|
||||
}
|
||||
|
||||
std::memcpy(copyTensorBuffer, tensorBuffer, inputTensor->get_byte_size());
|
||||
}
|
||||
}
|
||||
|
||||
_pipeline->push();
|
||||
}
|
||||
|
||||
void ZeroInferRequest::get_result() {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "get_result");
|
||||
|
||||
_pipeline->pull();
|
||||
|
||||
for (const auto& name : _outputAndStateOutputNames) {
|
||||
const std::shared_ptr<ov::ITensor>& outputTensor = _allTensors.at(name);
|
||||
const std::shared_ptr<ov::ITensor>& wrapperOutputTensor = _copyAllTensors.at(name);
|
||||
|
||||
uint8_t* tensorBuffer = reinterpret_cast<uint8_t*>(outputTensor->data());
|
||||
const uint8_t* copyTensorBuffer = reinterpret_cast<uint8_t*>(wrapperOutputTensor->data());
|
||||
|
||||
if (tensorBuffer != copyTensorBuffer) {
|
||||
if (tensorBuffer == nullptr || copyTensorBuffer == nullptr) {
|
||||
OPENVINO_THROW("Empty buffer");
|
||||
}
|
||||
|
||||
std::memcpy(tensorBuffer, copyTensorBuffer, outputTensor->get_byte_size());
|
||||
}
|
||||
}
|
||||
|
||||
_pipeline->reset();
|
||||
_logger.debug("InferRequest::get_result finished");
|
||||
}
|
||||
|
||||
void ZeroInferRequest::check_network_precision(const ov::element::Type_t precision) {
|
||||
switch (precision) {
|
||||
case ov::element::Type_t::f32:
|
||||
break;
|
||||
case ov::element::Type_t::f16:
|
||||
break;
|
||||
case ov::element::Type_t::u8:
|
||||
break;
|
||||
case ov::element::Type_t::i8:
|
||||
break;
|
||||
case ov::element::Type_t::u16:
|
||||
break;
|
||||
case ov::element::Type_t::i16:
|
||||
break;
|
||||
case ov::element::Type_t::u32:
|
||||
break;
|
||||
case ov::element::Type_t::i32:
|
||||
break;
|
||||
case ov::element::Type_t::u64:
|
||||
break;
|
||||
case ov::element::Type_t::i64:
|
||||
break;
|
||||
default:
|
||||
OPENVINO_THROW("Unsupported tensor precision: " + ov::element::Type(precision).get_type_name() +
|
||||
"! Supported precisions: FP32, FP16, U8, I8, U16, I16, U32, I32, U64, I64");
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<ov::ProfilingInfo> ZeroInferRequest::get_profiling_info() const {
|
||||
const auto& compiledModel = *std::dynamic_pointer_cast<const ICompiledModel>(_compiledModel);
|
||||
const auto& compilerConfig = compiledModel.get_config();
|
||||
if (!compilerConfig.get<PERF_COUNT>() || !_config.get<PERF_COUNT>()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
auto compilerType = compilerConfig.get<COMPILER_TYPE>();
|
||||
if (compilerType == ov::intel_npu::CompilerType::MLIR) {
|
||||
// For plugin compiler retreive raw profiling data from backend and delegate
|
||||
// processing to the compiler
|
||||
const auto& networkDesc = compiledModel.get_network_description();
|
||||
const auto& compiler = compiledModel.get_compiler();
|
||||
const auto& blob = networkDesc->compiledNetwork;
|
||||
auto profData = get_raw_profiling_data();
|
||||
return compiler->process_profiling_output(profData, blob, compilerConfig);
|
||||
} else {
|
||||
auto proftype = _config.get<PROFILING_TYPE>();
|
||||
if (proftype == ov::intel_npu::ProfilingType::INFER) {
|
||||
return _npu_profiling->getNpuInferStatistics();
|
||||
} else { /// proftype = MODEL or undefined = fallback to model profiling
|
||||
return _profiling_query.getLayerStatistics();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<uint8_t> ZeroInferRequest::get_raw_profiling_data() const {
|
||||
return _profiling_query.getData<uint8_t>();
|
||||
}
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "zero_init.hpp"
|
||||
|
||||
#include "intel_npu/al/itt.hpp"
|
||||
#include "zero_utils.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
const ze_driver_uuid_t ZeroInitStructsHolder::uuid = ze_intel_vpu_driver_uuid;
|
||||
|
||||
static std::tuple<uint32_t, std::string> queryDriverExtensionVersion(ze_driver_handle_t _driverHandle) {
|
||||
// query the extension properties
|
||||
uint32_t count = 0;
|
||||
zeroUtils::throwOnFail("zeDriverGetExtensionProperties",
|
||||
zeDriverGetExtensionProperties(_driverHandle, &count, nullptr));
|
||||
|
||||
std::vector<ze_driver_extension_properties_t> extProps;
|
||||
extProps.resize(count);
|
||||
zeroUtils::throwOnFail("zeDriverGetExtensionProperties",
|
||||
zeDriverGetExtensionProperties(_driverHandle, &count, extProps.data()));
|
||||
|
||||
const char* graphExtName = nullptr;
|
||||
uint32_t targetVersion = 0;
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
auto& property = extProps[i];
|
||||
|
||||
if (strncmp(property.name, ZE_GRAPH_EXT_NAME, strlen(ZE_GRAPH_EXT_NAME)) != 0)
|
||||
continue;
|
||||
|
||||
// If the driver version is latest, will just use its name.
|
||||
if (property.version == ZE_GRAPH_EXT_VERSION_CURRENT) {
|
||||
graphExtName = property.name;
|
||||
targetVersion = property.version;
|
||||
break;
|
||||
}
|
||||
|
||||
// Use the latest version supported by the driver.
|
||||
if (property.version > targetVersion) {
|
||||
graphExtName = property.name;
|
||||
targetVersion = property.version;
|
||||
}
|
||||
}
|
||||
|
||||
if (graphExtName == nullptr) {
|
||||
OPENVINO_THROW("queryDriverExtensionVersion: Failed to find Graph extension in NPU Driver");
|
||||
}
|
||||
|
||||
const uint16_t supportedDriverExtMajorVersion = 1;
|
||||
const uint16_t driverExtMajorVersion = ZE_MAJOR_VERSION(targetVersion);
|
||||
if (supportedDriverExtMajorVersion != driverExtMajorVersion) {
|
||||
OPENVINO_THROW("Plugin supports only driver with extension major version ",
|
||||
supportedDriverExtMajorVersion,
|
||||
"; discovered driver extension has major version ",
|
||||
driverExtMajorVersion);
|
||||
}
|
||||
|
||||
return std::make_tuple(targetVersion, graphExtName);
|
||||
}
|
||||
|
||||
ZeroInitStructsHolder::ZeroInitStructsHolder() : log("NPUZeroInitStructsHolder", Logger::global().level()) {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "ZeroInitStructsHolder::ZeroInitStructsHolder");
|
||||
zeroUtils::throwOnFail("zeInit", zeInit(ZE_INIT_FLAG_VPU_ONLY));
|
||||
|
||||
uint32_t drivers = 0;
|
||||
zeroUtils::throwOnFail("zeDriverGet", zeDriverGet(&drivers, nullptr));
|
||||
|
||||
std::vector<ze_driver_handle_t> all_drivers(drivers);
|
||||
zeroUtils::throwOnFail("zeDriverGet", zeDriverGet(&drivers, all_drivers.data()));
|
||||
|
||||
// Get our target driver
|
||||
ze_driver_properties_t props = {};
|
||||
props.stype = ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES;
|
||||
for (uint32_t i = 0; i < drivers; ++i) {
|
||||
zeDriverGetProperties(all_drivers[i], &props);
|
||||
|
||||
if (memcmp(&props.uuid, &uuid, sizeof(uuid)) == 0) {
|
||||
driver_handle = all_drivers[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (driver_handle == nullptr) {
|
||||
OPENVINO_THROW("zeDriverGet failed to return NPU driver");
|
||||
}
|
||||
|
||||
// Check L0 API version
|
||||
ze_api_version_t ze_drv_api_version = {};
|
||||
zeroUtils::throwOnFail("zeDriverGetApiVersion", zeDriverGetApiVersion(driver_handle, &ze_drv_api_version));
|
||||
|
||||
if (ZE_MAJOR_VERSION(ZE_API_VERSION_CURRENT) != ZE_MAJOR_VERSION(ze_drv_api_version)) {
|
||||
OPENVINO_THROW("Incompatibility between NPU plugin and driver! ",
|
||||
"Plugin L0 API major version = ",
|
||||
ZE_MAJOR_VERSION(ZE_API_VERSION_CURRENT),
|
||||
", ",
|
||||
"Driver L0 API major version = ",
|
||||
ZE_MAJOR_VERSION(ze_drv_api_version));
|
||||
}
|
||||
if (ZE_MINOR_VERSION(ZE_API_VERSION_CURRENT) != ZE_MINOR_VERSION(ze_drv_api_version)) {
|
||||
log.debug("Some features might not be available! "
|
||||
"Plugin L0 API minor version = %d, Driver L0 API minor version = %d",
|
||||
ZE_MINOR_VERSION(ZE_API_VERSION_CURRENT),
|
||||
ZE_MINOR_VERSION(ze_drv_api_version));
|
||||
}
|
||||
|
||||
// Query our graph extension version
|
||||
uint32_t driverExtVersion = 0;
|
||||
std::string graphExtName;
|
||||
std::tie(driverExtVersion, graphExtName) = queryDriverExtensionVersion(driver_handle);
|
||||
|
||||
log.debug("Found Driver Version %d.%d, Driver Extension Version %d.%d (%s)",
|
||||
ZE_MAJOR_VERSION(ze_drv_api_version),
|
||||
ZE_MINOR_VERSION(ze_drv_api_version),
|
||||
ZE_MAJOR_VERSION(driverExtVersion),
|
||||
ZE_MINOR_VERSION(driverExtVersion),
|
||||
graphExtName.c_str());
|
||||
|
||||
// Load our graph extension
|
||||
ze_graph_dditable_ext_last_t* graph_ddi_table_ext = nullptr;
|
||||
zeroUtils::throwOnFail("zeDriverGetExtensionFunctionAddress",
|
||||
zeDriverGetExtensionFunctionAddress(driver_handle,
|
||||
graphExtName.c_str(),
|
||||
reinterpret_cast<void**>(&graph_ddi_table_ext)));
|
||||
graph_dditable_ext_decorator =
|
||||
std::make_unique<ze_graph_dditable_ext_decorator>(graph_ddi_table_ext, driverExtVersion);
|
||||
|
||||
// Load our profiling extension
|
||||
zeroUtils::throwOnFail(
|
||||
"zeDriverGetExtensionFunctionAddress",
|
||||
zeDriverGetExtensionFunctionAddress(driver_handle,
|
||||
"ZE_extension_profiling_data",
|
||||
reinterpret_cast<void**>(&_graph_profiling_ddi_table_ext)));
|
||||
|
||||
uint32_t device_count = 1;
|
||||
// Get our target device
|
||||
zeroUtils::throwOnFail("zeDeviceGet", zeDeviceGet(driver_handle, &device_count, &device_handle));
|
||||
|
||||
ze_context_desc_t context_desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, 0, 0};
|
||||
zeroUtils::throwOnFail("zeContextCreate", zeContextCreate(driver_handle, &context_desc, &context));
|
||||
}
|
||||
|
||||
ZeroInitStructsHolder::~ZeroInitStructsHolder() {
|
||||
if (context) {
|
||||
auto result = zeContextDestroy(context);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
log.error("zeContextDestroy failed %#X", uint64_t(result));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "zero_memory.hpp"
|
||||
|
||||
#include "zero_utils.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
namespace zeroMemory {
|
||||
DeviceMem::DeviceMem(const ze_device_handle_t device_handle, ze_context_handle_t context, const std::size_t size)
|
||||
: _size(size),
|
||||
_context(context),
|
||||
_log("DeviceMem", Logger::global().level()) {
|
||||
ze_device_mem_alloc_desc_t desc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, nullptr, 0, 0};
|
||||
|
||||
zeroUtils::throwOnFail("zeMemAllocDevice",
|
||||
zeMemAllocDevice(_context, &desc, _size, _alignment, device_handle, &_data));
|
||||
}
|
||||
DeviceMem& DeviceMem::operator=(DeviceMem&& other) {
|
||||
if (this == &other)
|
||||
return *this;
|
||||
free();
|
||||
_size = other._size;
|
||||
_data = other._data;
|
||||
_context = other._context;
|
||||
other._size = 0;
|
||||
other._data = nullptr;
|
||||
return *this;
|
||||
}
|
||||
void DeviceMem::free() {
|
||||
if (_size != 0) {
|
||||
_size = 0;
|
||||
zeroUtils::throwOnFail("zeMemFree DeviceMem", zeMemFree(_context, _data));
|
||||
_data = nullptr;
|
||||
}
|
||||
}
|
||||
DeviceMem::~DeviceMem() {
|
||||
try {
|
||||
free();
|
||||
} catch (const std::exception& e) {
|
||||
_log.error("Caught when freeing memory: %s", e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void* HostMemAllocator::allocate(const size_t bytes, const size_t /*alignment*/) noexcept {
|
||||
size_t size = bytes + _alignment - (bytes % _alignment);
|
||||
|
||||
ze_host_mem_alloc_desc_t desc = {ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC,
|
||||
nullptr,
|
||||
static_cast<ze_host_mem_alloc_flags_t>(_flag)};
|
||||
ze_result_t res = zeMemAllocHost(_initStructs->getContext(), &desc, size, _alignment, &_data);
|
||||
|
||||
if (res == ZE_RESULT_SUCCESS) {
|
||||
return _data;
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
bool HostMemAllocator::deallocate(void* handle, const size_t /* bytes */, size_t /* alignment */) noexcept {
|
||||
auto result = zeMemFree(_initStructs->getContext(), handle);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
bool HostMemAllocator::is_equal(const HostMemAllocator& other) const {
|
||||
return other._data != nullptr && _data != nullptr && other._data == _data;
|
||||
}
|
||||
|
||||
void MemoryManagementUnit::appendArgument(const std::string& name, const ze_graph_argument_properties_t& argument) {
|
||||
_offsets.emplace(std::make_pair(name, _size));
|
||||
|
||||
const std::size_t argSize = zeroUtils::getSizeIOBytes(argument);
|
||||
_size += argSize + alignment -
|
||||
(argSize % alignment); // is this really necessary? if 0==argSize%alignment -> add 1 * alignment
|
||||
}
|
||||
|
||||
void MemoryManagementUnit::allocate(const ze_device_handle_t device_handle, const ze_context_handle_t context) {
|
||||
if (_size == 0) {
|
||||
OPENVINO_THROW("Can't allocate empty buffer");
|
||||
}
|
||||
|
||||
_device = std::make_unique<DeviceMem>(device_handle, context, _size);
|
||||
}
|
||||
std::size_t MemoryManagementUnit::getSize() const {
|
||||
return _size;
|
||||
}
|
||||
const void* MemoryManagementUnit::getDeviceMemRegion() const {
|
||||
return _device ? _device->data() : nullptr;
|
||||
}
|
||||
void* MemoryManagementUnit::getDeviceMemRegion() {
|
||||
return _device ? _device->data() : nullptr;
|
||||
}
|
||||
void* MemoryManagementUnit::getDevicePtr(const std::string& name) {
|
||||
uint8_t* from = static_cast<uint8_t*>(_device ? _device->data() : nullptr);
|
||||
if (from == nullptr) {
|
||||
OPENVINO_THROW("Device memory not allocated yet");
|
||||
}
|
||||
if (!_offsets.count(name)) {
|
||||
OPENVINO_THROW("Invalid memory offset key: ", name);
|
||||
}
|
||||
|
||||
return _offsets.at(name) + from;
|
||||
}
|
||||
} // namespace zeroMemory
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,288 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "zero_pipeline.hpp"
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include "intel_npu/al/itt.hpp"
|
||||
#include "intel_npu/al/prefix.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
struct DiscretePipeline final : public Pipeline {
|
||||
public:
|
||||
DiscretePipeline(const Config& config,
|
||||
const ze_device_handle_t& device_handle,
|
||||
ze_context_handle_t& context,
|
||||
ze_graph_dditable_ext_curr_t* graph_ddi_table_ext,
|
||||
const std::shared_ptr<const IExecutor>& executorPtr,
|
||||
ze_graph_profiling_query_handle_t profiling_handle,
|
||||
const std::array<std::shared_ptr<CommandQueue>, stage::COUNT>& command_queues,
|
||||
const uint32_t& group_ordinal,
|
||||
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors)
|
||||
: _config(config),
|
||||
_command_queues{command_queues},
|
||||
_command_list{{{device_handle, context, graph_ddi_table_ext, _config, group_ordinal},
|
||||
{device_handle, context, graph_ddi_table_ext, _config, group_ordinal},
|
||||
{device_handle, context, graph_ddi_table_ext, _config, group_ordinal}}},
|
||||
_fence{{{*_command_queues[stage::UPLOAD], _config},
|
||||
{*_command_queues[stage::EXECUTE], _config},
|
||||
{*_command_queues[stage::READBACK], _config}}},
|
||||
_event_pool(device_handle, context, stage::COUNT, _config),
|
||||
_event{{{_event_pool.handle(), stage::UPLOAD, _config},
|
||||
{_event_pool.handle(), stage::EXECUTE, _config},
|
||||
{_event_pool.handle(), stage::READBACK, _config}}} {
|
||||
const ZeroExecutor* executor = static_cast<const ZeroExecutor*>(executorPtr.get());
|
||||
static const std::size_t alignment = STANDARD_PAGE_SIZE;
|
||||
|
||||
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Zero_infer_request::DiscretePipeline::DiscretePipeline");
|
||||
for (const auto& desc : executor->inputs_desc_map()) {
|
||||
_deviceInputs.appendArgument(desc.first, desc.second.info);
|
||||
}
|
||||
_deviceInputs.allocate(device_handle, context);
|
||||
|
||||
for (const auto& desc : executor->inputs_desc_map()) {
|
||||
const std::shared_ptr<ov::ITensor>& inputTensor = tensors.at(desc.first);
|
||||
const void* tensorBuffer = reinterpret_cast<const void*>(inputTensor->data());
|
||||
|
||||
const std::size_t argSize = zeroUtils::getSizeIOBytes(desc.second.info);
|
||||
std::size_t size = argSize + alignment - (argSize % alignment);
|
||||
|
||||
_command_list[stage::UPLOAD].appendMemoryCopy(_deviceInputs.getDevicePtr(desc.first), tensorBuffer, size);
|
||||
|
||||
executor->setArgumentValue(desc.second.idx, _deviceInputs.getDevicePtr(desc.first));
|
||||
}
|
||||
|
||||
_command_list[stage::UPLOAD].appendBarrier();
|
||||
_event[stage::UPLOAD].AppendSignalEvent(_command_list[stage::UPLOAD]);
|
||||
|
||||
for (const auto& desc : executor->outputs_desc_map()) {
|
||||
_deviceOutputs.appendArgument(desc.first, desc.second.info);
|
||||
}
|
||||
_deviceOutputs.allocate(device_handle, context);
|
||||
|
||||
for (const auto& desc : executor->outputs_desc_map()) {
|
||||
const std::shared_ptr<ov::ITensor>& outputTensor = tensors.at(desc.first);
|
||||
void* tensorBuffer = reinterpret_cast<void*>(outputTensor->data());
|
||||
|
||||
const std::size_t argSize = zeroUtils::getSizeIOBytes(desc.second.info);
|
||||
std::size_t size = argSize + alignment - (argSize % alignment);
|
||||
|
||||
_command_list[stage::READBACK].appendMemoryCopy(tensorBuffer,
|
||||
_deviceOutputs.getDevicePtr(desc.first),
|
||||
size);
|
||||
|
||||
executor->setArgumentValue(desc.second.idx, _deviceOutputs.getDevicePtr(desc.first));
|
||||
}
|
||||
|
||||
_event[stage::UPLOAD].AppendWaitOnEvent(_command_list[stage::EXECUTE]);
|
||||
|
||||
_command_list[stage::EXECUTE].appendGraphExecute(executor->graph(), profiling_handle);
|
||||
|
||||
_event[stage::UPLOAD].AppendEventReset(_command_list[stage::READBACK]);
|
||||
|
||||
for (auto& commandList : _command_list) {
|
||||
commandList.close();
|
||||
}
|
||||
};
|
||||
|
||||
DiscretePipeline(const DiscretePipeline&) = delete;
|
||||
DiscretePipeline& operator=(const DiscretePipeline&) = delete;
|
||||
virtual ~DiscretePipeline() = default;
|
||||
|
||||
void push() override {
|
||||
OV_ITT_TASK_CHAIN(ZERO_INFER_REQUEST_DP_PUSH,
|
||||
itt::domains::LevelZeroBackend,
|
||||
"DiscretePipeline::push",
|
||||
"UPLOAD");
|
||||
// Dispatch command to copy input data from upload heap to default heap
|
||||
_command_queues[stage::UPLOAD]->executeCommandList(_command_list[stage::UPLOAD]);
|
||||
|
||||
OV_ITT_TASK_NEXT(ZERO_INFER_REQUEST_DP_PUSH, "EXECUTE");
|
||||
// Submit the command list for execute
|
||||
_command_queues[stage::EXECUTE]->executeCommandList(_command_list[stage::EXECUTE], _fence[stage::EXECUTE]);
|
||||
};
|
||||
|
||||
void pull() override {
|
||||
OV_ITT_TASK_CHAIN(ZERO_INFER_REQUEST_DP_PULL,
|
||||
itt::domains::LevelZeroBackend,
|
||||
"DiscretePipeline::pull",
|
||||
"EXECUTE");
|
||||
// Wait for execute to finish
|
||||
_fence[stage::EXECUTE].hostSynchronize();
|
||||
OV_ITT_TASK_NEXT(ZERO_INFER_REQUEST_DP_PULL, "READBACK");
|
||||
// Schedule the copy of outputs from zeDriverAllocDeviceMem to zeDriverAllocHostMem
|
||||
_command_queues[stage::READBACK]->executeCommandList(_command_list[stage::READBACK], _fence[stage::READBACK]);
|
||||
// Wait for output copy to finish execution for _fence from the host, to make sure that data
|
||||
// is available in the hostMem buffer of the output
|
||||
_fence[stage::READBACK].hostSynchronize();
|
||||
};
|
||||
|
||||
void reset() const override {
|
||||
// Reset the fence objects
|
||||
for (auto& fence : _fence) {
|
||||
fence.reset();
|
||||
}
|
||||
};
|
||||
|
||||
private:
|
||||
const Config _config;
|
||||
const std::array<std::shared_ptr<CommandQueue>, stage::COUNT>& _command_queues;
|
||||
std::array<CommandList, stage::COUNT> _command_list;
|
||||
std::array<Fence, stage::COUNT> _fence;
|
||||
EventPool _event_pool;
|
||||
std::array<Event, stage::COUNT> _event;
|
||||
};
|
||||
|
||||
struct IntegratedPipeline final : public Pipeline {
|
||||
public:
|
||||
IntegratedPipeline(const Config& config,
|
||||
const ze_device_handle_t& device_handle,
|
||||
ze_context_handle_t& context,
|
||||
ze_graph_dditable_ext_curr_t* graph_ddi_table_ext,
|
||||
const std::shared_ptr<const IExecutor>& executorPtr,
|
||||
ze_graph_profiling_query_handle_t profiling_handle,
|
||||
std::shared_ptr<zeroProfiling::NpuInferProfiling> npu_profiling,
|
||||
CommandQueue& command_queue,
|
||||
const uint32_t& group_ordinal,
|
||||
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors)
|
||||
: _config(config),
|
||||
_command_queue{command_queue},
|
||||
_command_list{device_handle, context, graph_ddi_table_ext, _config, group_ordinal},
|
||||
_fence{_command_queue, _config},
|
||||
_event_pool{device_handle, context, 1, _config},
|
||||
_event{_event_pool.handle(), 0, _config},
|
||||
_npu_profiling(npu_profiling) {
|
||||
const ZeroExecutor* executor = static_cast<const ZeroExecutor*>(executorPtr.get());
|
||||
|
||||
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend,
|
||||
"Zero_infer_request::IntegratedPipeline::IntegratedPipeline");
|
||||
|
||||
for (const auto& desc : executor->inputs_desc_map()) {
|
||||
const std::shared_ptr<ov::ITensor>& inputTensor = tensors.at(desc.first);
|
||||
executor->setArgumentValue(desc.second.idx, inputTensor->data());
|
||||
}
|
||||
|
||||
for (const auto& desc : executor->outputs_desc_map()) {
|
||||
const std::shared_ptr<ov::ITensor>& outputTensor = tensors.at(desc.first);
|
||||
executor->setArgumentValue(desc.second.idx, outputTensor->data());
|
||||
}
|
||||
|
||||
/// append timestamp command if feature was activated
|
||||
if (_npu_profiling != nullptr) {
|
||||
_command_list.appendBarrier();
|
||||
_command_list.appendNpuTimestamp(reinterpret_cast<uint64_t*>(_npu_profiling->npu_ts_infer_start));
|
||||
}
|
||||
|
||||
_command_list.appendGraphExecute(executor->graph(), profiling_handle);
|
||||
|
||||
/// append timestamp command if feature was activated
|
||||
if (_npu_profiling != nullptr) {
|
||||
_command_list.appendBarrier();
|
||||
_command_list.appendNpuTimestamp(reinterpret_cast<uint64_t*>(_npu_profiling->npu_ts_infer_end));
|
||||
}
|
||||
|
||||
// appendBarrier used in L0 as well
|
||||
if (!sync_output_with_fences_) {
|
||||
_command_list.appendBarrier();
|
||||
_event.AppendSignalEvent(_command_list);
|
||||
}
|
||||
_command_list.close();
|
||||
}
|
||||
|
||||
IntegratedPipeline(const IntegratedPipeline&) = delete;
|
||||
IntegratedPipeline& operator=(const IntegratedPipeline&) = delete;
|
||||
virtual ~IntegratedPipeline() = default;
|
||||
|
||||
void push() override {
|
||||
OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_IP_PUSH, itt::domains::LevelZeroBackend, "IntegratedPipeline", "push");
|
||||
if (sync_output_with_fences_) {
|
||||
_command_queue.executeCommandList(_command_list, _fence);
|
||||
} else {
|
||||
_command_queue.executeCommandList(_command_list);
|
||||
}
|
||||
};
|
||||
|
||||
void pull() override {
|
||||
OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_IP_PULL, itt::domains::LevelZeroBackend, "IntegratedPipeline", "pull");
|
||||
if (sync_output_with_fences_) {
|
||||
_fence.hostSynchronize();
|
||||
} else {
|
||||
_event.hostSynchronize();
|
||||
}
|
||||
/// sample npu timestamps if feature was activated
|
||||
if (_npu_profiling != nullptr) {
|
||||
_npu_profiling->sampleNpuTimestamps();
|
||||
}
|
||||
};
|
||||
|
||||
void reset() const override {
|
||||
if (sync_output_with_fences_) {
|
||||
_fence.reset();
|
||||
} else {
|
||||
_event.reset();
|
||||
}
|
||||
};
|
||||
|
||||
private:
|
||||
const Config _config;
|
||||
CommandQueue& _command_queue;
|
||||
CommandList _command_list;
|
||||
Fence _fence;
|
||||
EventPool _event_pool;
|
||||
Event _event;
|
||||
bool sync_output_with_fences_ = true;
|
||||
std::shared_ptr<zeroProfiling::NpuInferProfiling> _npu_profiling;
|
||||
};
|
||||
|
||||
std::unique_ptr<Pipeline> makePipeline(const std::shared_ptr<const IExecutor>& executorPtr,
|
||||
const Config& config,
|
||||
zeroProfiling::ProfilingPool& profiling_pool,
|
||||
zeroProfiling::ProfilingQuery& profiling_query,
|
||||
std::shared_ptr<zeroProfiling::NpuInferProfiling> npu_profiling,
|
||||
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors) {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Infer_request::makePipeline");
|
||||
if (profiling_pool.create())
|
||||
profiling_query.create(profiling_pool._handle);
|
||||
|
||||
const ZeroExecutor* executor = static_cast<const ZeroExecutor*>(executorPtr.get());
|
||||
|
||||
const ze_device_handle_t device_handle = executor->getInitStructs()->getDevice();
|
||||
ze_context_handle_t context = executor->getInitStructs()->getContext();
|
||||
ze_graph_dditable_ext_curr_t* graph_ddi_table_ext = executor->getInitStructs()->getGraphDdiTable();
|
||||
auto& command_queues = executor->getCommandQueue();
|
||||
uint32_t group_ordinal = executor->get_group_ordinal();
|
||||
|
||||
ze_device_properties_t properties = {};
|
||||
properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
|
||||
zeroUtils::throwOnFail("zeDeviceGetProperties", zeDeviceGetProperties(device_handle, &properties));
|
||||
|
||||
if (properties.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) {
|
||||
return std::make_unique<IntegratedPipeline>(config,
|
||||
device_handle,
|
||||
context,
|
||||
graph_ddi_table_ext,
|
||||
executorPtr,
|
||||
profiling_query.getHandle(),
|
||||
npu_profiling,
|
||||
*command_queues[stage::EXECUTE],
|
||||
group_ordinal,
|
||||
tensors);
|
||||
}
|
||||
|
||||
return std::make_unique<DiscretePipeline>(config,
|
||||
device_handle,
|
||||
context,
|
||||
graph_ddi_table_ext,
|
||||
executorPtr,
|
||||
profiling_query.getHandle(),
|
||||
command_queues,
|
||||
group_ordinal,
|
||||
tensors);
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,249 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "zero_profiling.hpp"
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_profiling_ext.h>
|
||||
|
||||
#include "intel_npu/al/config/compiler.hpp"
|
||||
#include "intel_npu/al/profiling.hpp"
|
||||
#include "zero_profiling.hpp"
|
||||
#include "zero_utils.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
namespace zeroProfiling {
|
||||
|
||||
/// @brief Type trait mapping from ZE data type to enum value
|
||||
template <typename T>
|
||||
struct ZeProfilingTypeId {};
|
||||
|
||||
template <>
|
||||
struct ZeProfilingTypeId<ze_profiling_layer_info> {
|
||||
static const ze_graph_profiling_type_t value = ZE_GRAPH_PROFILING_LAYER_LEVEL;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ZeProfilingTypeId<ze_profiling_task_info> {
|
||||
static const ze_graph_profiling_type_t value = ZE_GRAPH_PROFILING_TASK_LEVEL;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct ZeProfilingTypeId<uint8_t> {
|
||||
static const ze_graph_profiling_type_t value = ZE_GRAPH_PROFILING_RAW;
|
||||
};
|
||||
|
||||
bool ProfilingPool::create() {
|
||||
auto ret = _graph_profiling_ddi_table_ext->pfnProfilingPoolCreate(_graph_handle, _profiling_count, &_handle);
|
||||
return ((ZE_RESULT_SUCCESS == ret) && (_handle != nullptr));
|
||||
}
|
||||
|
||||
ProfilingPool::~ProfilingPool() {
|
||||
if (_handle) {
|
||||
_graph_profiling_ddi_table_ext->pfnProfilingPoolDestroy(_handle);
|
||||
}
|
||||
}
|
||||
|
||||
void ProfilingQuery::create(const ze_graph_profiling_pool_handle_t& profiling_pool) {
|
||||
zeroUtils::throwOnFail("pfnProfilingQueryCreate",
|
||||
_graph_profiling_ddi_table_ext->pfnProfilingQueryCreate(profiling_pool, _index, &_handle));
|
||||
}
|
||||
|
||||
LayerStatistics ProfilingQuery::getLayerStatistics() const {
|
||||
verifyProfilingProperties();
|
||||
auto layerData = getData<ze_profiling_layer_info>();
|
||||
return profiling::convertLayersToIeProfilingInfo(layerData);
|
||||
}
|
||||
|
||||
ProfilingQuery::~ProfilingQuery() {
|
||||
if (_handle) {
|
||||
_graph_profiling_ddi_table_ext->pfnProfilingQueryDestroy(_handle);
|
||||
}
|
||||
}
|
||||
|
||||
void ProfilingQuery::queryGetData(const ze_graph_profiling_type_t profilingType,
|
||||
uint32_t* pSize,
|
||||
uint8_t* pData) const {
|
||||
if (_handle && pSize) {
|
||||
zeroUtils::throwOnFail(
|
||||
"pfnProfilingQueryGetData",
|
||||
_graph_profiling_ddi_table_ext->pfnProfilingQueryGetData(_handle, profilingType, pSize, pData));
|
||||
}
|
||||
}
|
||||
|
||||
template <class ProfilingData>
|
||||
std::vector<ProfilingData> ProfilingQuery::getData() const {
|
||||
ze_graph_profiling_type_t type = ZeProfilingTypeId<ProfilingData>::value;
|
||||
uint32_t size = 0;
|
||||
|
||||
// Obtain the size of the buffer
|
||||
queryGetData(type, &size, nullptr);
|
||||
|
||||
OPENVINO_ASSERT(size % sizeof(ProfilingData) == 0);
|
||||
|
||||
// Allocate enough memory and copy the buffer
|
||||
std::vector<ProfilingData> profilingData(size / sizeof(ProfilingData));
|
||||
queryGetData(type, &size, reinterpret_cast<uint8_t*>(profilingData.data()));
|
||||
return profilingData;
|
||||
}
|
||||
|
||||
template std::vector<uint8_t> ProfilingQuery::getData<uint8_t>() const;
|
||||
|
||||
void ProfilingQuery::getProfilingProperties(ze_device_profiling_data_properties_t* properties) const {
|
||||
if (_handle && properties) {
|
||||
zeroUtils::throwOnFail(
|
||||
"getProfilingProperties",
|
||||
_graph_profiling_ddi_table_ext->pfnDeviceGetProfilingDataProperties(_device_handle, properties));
|
||||
}
|
||||
}
|
||||
|
||||
void ProfilingQuery::verifyProfilingProperties() const {
|
||||
if (!_handle) {
|
||||
OPENVINO_THROW("Can't get profiling statistics because profiling is disabled.");
|
||||
}
|
||||
const auto stringifyVersion = [](auto version) -> std::string {
|
||||
return std::to_string(ZE_MAJOR_VERSION(version)) + "." + std::to_string(ZE_MINOR_VERSION(version));
|
||||
};
|
||||
|
||||
ze_device_profiling_data_properties_t profProp;
|
||||
getProfilingProperties(&profProp);
|
||||
const auto currentProfilingVersion = ze_profiling_data_ext_version_t::ZE_PROFILING_DATA_EXT_VERSION_CURRENT;
|
||||
|
||||
if (ZE_MAJOR_VERSION(profProp.extensionVersion) != ZE_MAJOR_VERSION(currentProfilingVersion)) {
|
||||
OPENVINO_THROW("Unsupported NPU driver.",
|
||||
"Profiling API version: plugin: ",
|
||||
stringifyVersion(currentProfilingVersion),
|
||||
", driver: ",
|
||||
stringifyVersion(profProp.extensionVersion));
|
||||
}
|
||||
if (currentProfilingVersion > profProp.extensionVersion) {
|
||||
auto log = Logger::global().clone("ZeroProfilingQuery");
|
||||
log.warning("Outdated NPU driver detected. Some features might not be available! "
|
||||
"Profiling API version: plugin: %s, driver: %s",
|
||||
stringifyVersion(currentProfilingVersion).c_str(),
|
||||
stringifyVersion(profProp.extensionVersion).c_str());
|
||||
}
|
||||
}
|
||||
|
||||
NpuInferStatistics NpuInferProfiling::getNpuInferStatistics() const {
|
||||
NpuInferStatistics npuPerfCounts;
|
||||
|
||||
/// if the log isn't full/rolled over yet = skip reporting empty logs
|
||||
uint32_t stat_cnt = (_npu_infer_stats_cnt < _npu_infer_log_maxsize) ? _npu_infer_stats_cnt : _npu_infer_log_maxsize;
|
||||
if (stat_cnt != 0 && _loglevel >= ov::log::Level::WARNING) {
|
||||
/// Populate npuinferstatistics vector
|
||||
for (unsigned i = 0; i < stat_cnt; i++) {
|
||||
ov::ProfilingInfo info;
|
||||
|
||||
info.status = ov::ProfilingInfo::Status::EXECUTED;
|
||||
info.real_time = std::chrono::microseconds(convertCCtoUS(_npu_infer_duration_log[i]));
|
||||
info.cpu_time = std::chrono::microseconds(convertCCtoUS(_npu_infer_duration_log[i]));
|
||||
info.node_name = std::to_string(i);
|
||||
info.exec_type = "INFER_REQ";
|
||||
info.node_type = "INFER_REQ";
|
||||
|
||||
npuPerfCounts.push_back(info);
|
||||
}
|
||||
}
|
||||
|
||||
/// sanity check to avoid division by 0
|
||||
if (_npu_infer_stats_cnt == 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
/// Add final statistics
|
||||
ov::ProfilingInfo info_avg = {
|
||||
ov::ProfilingInfo::Status::EXECUTED,
|
||||
std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_accu_cc / _npu_infer_stats_cnt)),
|
||||
std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_accu_cc / _npu_infer_stats_cnt)),
|
||||
"AVG",
|
||||
"AVG",
|
||||
"AVG"};
|
||||
npuPerfCounts.push_back(info_avg);
|
||||
ov::ProfilingInfo info_min = {ov::ProfilingInfo::Status::EXECUTED,
|
||||
std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_min_cc)),
|
||||
std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_min_cc)),
|
||||
"MIN",
|
||||
"MIN",
|
||||
"MIN"};
|
||||
npuPerfCounts.push_back(info_min);
|
||||
ov::ProfilingInfo info_max = {ov::ProfilingInfo::Status::EXECUTED,
|
||||
std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_max_cc)),
|
||||
std::chrono::microseconds(convertCCtoUS(_npu_infer_stats_max_cc)),
|
||||
"MAX",
|
||||
"MAX",
|
||||
"MAX"};
|
||||
npuPerfCounts.push_back(info_max);
|
||||
return npuPerfCounts;
|
||||
}
|
||||
|
||||
NpuInferProfiling::NpuInferProfiling(ze_context_handle_t context,
|
||||
ze_device_handle_t device_handle,
|
||||
ov::log::Level loglevel)
|
||||
: _context(context),
|
||||
_device_handle(device_handle),
|
||||
_loglevel(loglevel),
|
||||
_logger("InferProfiling", loglevel) {
|
||||
/// Fetch and store the device timer resolution
|
||||
_dev_properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES_1_2;
|
||||
zeroUtils::throwOnFail("zeDeviceGetProperties", zeDeviceGetProperties(_device_handle, &_dev_properties));
|
||||
/// Request mem allocations
|
||||
ze_host_mem_alloc_desc_t desc = {ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC,
|
||||
nullptr,
|
||||
ZE_HOST_MEM_ALLOC_FLAG_BIAS_CACHED};
|
||||
zeroUtils::throwOnFail("zeMemAllocHost",
|
||||
zeMemAllocHost(_context,
|
||||
&desc,
|
||||
sizeof(uint64_t),
|
||||
64,
|
||||
&npu_ts_infer_start)); // align to 64 bytes to match npu l2 cache line size
|
||||
zeroUtils::throwOnFail("zeMemAllocHost",
|
||||
zeMemAllocHost(_context,
|
||||
&desc,
|
||||
sizeof(uint64_t),
|
||||
64,
|
||||
&npu_ts_infer_end)); // alight to 64 bytes to match npu l2 cache line size
|
||||
}
|
||||
|
||||
void NpuInferProfiling::sampleNpuTimestamps() {
|
||||
int64_t infer_duration_cc = static_cast<int64_t>(*(reinterpret_cast<uint64_t*>(npu_ts_infer_end)) -
|
||||
*(reinterpret_cast<uint64_t*>(npu_ts_infer_start)));
|
||||
|
||||
/// Update extremas
|
||||
if (infer_duration_cc < _npu_infer_stats_min_cc)
|
||||
_npu_infer_stats_min_cc = infer_duration_cc;
|
||||
if (infer_duration_cc > _npu_infer_stats_max_cc)
|
||||
_npu_infer_stats_max_cc = infer_duration_cc;
|
||||
_npu_infer_stats_accu_cc += infer_duration_cc;
|
||||
_npu_infer_stats_cnt++;
|
||||
/// only log individual infer durations if requested
|
||||
if (_loglevel >= ov::log::Level::WARNING) {
|
||||
_npu_infer_duration_log[_npu_infer_logidx++] = infer_duration_cc;
|
||||
if (_npu_infer_logidx >= _npu_infer_log_maxsize)
|
||||
_npu_infer_logidx = 0;
|
||||
}
|
||||
}
|
||||
|
||||
int64_t NpuInferProfiling::convertCCtoUS(int64_t val_cc) const {
|
||||
return (int64_t)(val_cc * 1000 * 1000 / _dev_properties.timerResolution);
|
||||
}
|
||||
|
||||
NpuInferProfiling::~NpuInferProfiling() {
|
||||
/// deallocate npu_ts_infer_start and npu_ts_infer_end, allocated externally by ze driver
|
||||
if (npu_ts_infer_start != nullptr) {
|
||||
auto ze_ret = zeMemFree(_context, npu_ts_infer_start);
|
||||
if (ZE_RESULT_SUCCESS != ze_ret) {
|
||||
_logger.error("zeMemFree on npu_ts_infer_start failed %#X", uint64_t(ze_ret));
|
||||
}
|
||||
}
|
||||
if (npu_ts_infer_end != nullptr) {
|
||||
auto ze_ret = zeMemFree(_context, npu_ts_infer_end);
|
||||
if (ZE_RESULT_SUCCESS != ze_ret) {
|
||||
_logger.error("zeMemFree on npu_ts_infer_end failed %#X", uint64_t(ze_ret));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace zeroProfiling
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "zero_wrappers.hpp"
|
||||
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
EventPool::EventPool(ze_device_handle_t device_handle,
|
||||
const ze_context_handle_t& context,
|
||||
uint32_t event_count,
|
||||
const Config& config)
|
||||
: _log("EventPool", config.get<LOG_LEVEL>()) {
|
||||
ze_event_pool_desc_t event_pool_desc = {ZE_STRUCTURE_TYPE_EVENT_POOL_DESC,
|
||||
nullptr,
|
||||
ZE_EVENT_POOL_FLAG_HOST_VISIBLE,
|
||||
event_count};
|
||||
zeroUtils::throwOnFail("zeEventPoolCreate",
|
||||
zeEventPoolCreate(context, &event_pool_desc, 1, &device_handle, &_handle));
|
||||
}
|
||||
EventPool::~EventPool() {
|
||||
auto result = zeEventPoolDestroy(_handle);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
_log.error("zeEventPoolDestroy failed %#X", uint64_t(result));
|
||||
}
|
||||
}
|
||||
|
||||
Event::Event(const ze_event_pool_handle_t& event_pool, uint32_t event_index, const Config& config)
|
||||
: _log("Event", config.get<LOG_LEVEL>()) {
|
||||
ze_event_desc_t event_desc = {ZE_STRUCTURE_TYPE_EVENT_DESC, nullptr, event_index, 0, 0};
|
||||
zeroUtils::throwOnFail("zeEventCreate", zeEventCreate(event_pool, &event_desc, &_handle));
|
||||
}
|
||||
void Event::AppendSignalEvent(CommandList& command_list) const {
|
||||
zeroUtils::throwOnFail("zeCommandListAppendSignalEvent",
|
||||
zeCommandListAppendSignalEvent(command_list.handle(), _handle));
|
||||
}
|
||||
void Event::AppendWaitOnEvent(CommandList& command_list) {
|
||||
zeroUtils::throwOnFail("zeCommandListAppendWaitOnEvents",
|
||||
zeCommandListAppendWaitOnEvents(command_list.handle(), 1, &_handle));
|
||||
}
|
||||
void Event::AppendEventReset(CommandList& command_list) const {
|
||||
zeroUtils::throwOnFail("zeCommandListAppendEventReset",
|
||||
zeCommandListAppendEventReset(command_list.handle(), _handle));
|
||||
}
|
||||
void Event::hostSynchronize() const {
|
||||
zeroUtils::throwOnFail("zeEventHostSynchronize", zeEventHostSynchronize(_handle, UINT64_MAX));
|
||||
}
|
||||
void Event::reset() const {
|
||||
zeroUtils::throwOnFail("zeEventHostReset", zeEventHostReset(_handle));
|
||||
}
|
||||
Event::~Event() {
|
||||
auto result = zeEventDestroy(_handle);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
_log.error("zeEventDestroy failed %#X", uint64_t(result));
|
||||
}
|
||||
}
|
||||
|
||||
CommandList::CommandList(const ze_device_handle_t& device_handle,
|
||||
const ze_context_handle_t& context,
|
||||
ze_graph_dditable_ext_curr_t* graph_ddi_table_ext,
|
||||
const Config& config,
|
||||
const uint32_t& group_ordinal)
|
||||
: _context(context),
|
||||
_graph_ddi_table_ext(graph_ddi_table_ext),
|
||||
_log("CommandList", config.get<LOG_LEVEL>()) {
|
||||
ze_command_list_desc_t desc = {ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, nullptr, group_ordinal, 0};
|
||||
zeroUtils::throwOnFail("zeCommandListCreate", zeCommandListCreate(_context, device_handle, &desc, &_handle));
|
||||
}
|
||||
void CommandList::reset() const {
|
||||
zeroUtils::throwOnFail("zeCommandListReset", zeCommandListReset(_handle));
|
||||
}
|
||||
void CommandList::appendMemoryCopy(void* dst, const void* src, const std::size_t size) const {
|
||||
zeroUtils::throwOnFail("zeCommandListAppendMemoryCopy",
|
||||
zeCommandListAppendMemoryCopy(_handle, dst, src, size, nullptr, 0, nullptr));
|
||||
}
|
||||
void CommandList::appendGraphInitialize(const ze_graph_handle_t& graph_handle) const {
|
||||
zeroUtils::throwOnFail("pfnAppendGraphInitialize",
|
||||
_graph_ddi_table_ext->pfnAppendGraphInitialize(_handle, graph_handle, nullptr, 0, nullptr));
|
||||
}
|
||||
void CommandList::appendGraphExecute(const ze_graph_handle_t& graph_handle,
|
||||
const ze_graph_profiling_query_handle_t& profiling_query_handle) const {
|
||||
zeroUtils::throwOnFail(
|
||||
"pfnAppendGraphExecute",
|
||||
_graph_ddi_table_ext
|
||||
->pfnAppendGraphExecute(_handle, graph_handle, profiling_query_handle, nullptr, 0, nullptr));
|
||||
}
|
||||
void CommandList::appendNpuTimestamp(uint64_t* timestamp_buff) const {
|
||||
zeroUtils::throwOnFail("zeCommandListAppendWriteGlobalTimestamp",
|
||||
zeCommandListAppendWriteGlobalTimestamp(_handle, timestamp_buff, nullptr, 0, nullptr));
|
||||
}
|
||||
void CommandList::appendBarrier() const {
|
||||
zeroUtils::throwOnFail("zeCommandListAppendBarrier", zeCommandListAppendBarrier(_handle, nullptr, 0, nullptr));
|
||||
}
|
||||
void CommandList::close() const {
|
||||
zeroUtils::throwOnFail("zeCommandListClose", zeCommandListClose(_handle));
|
||||
}
|
||||
CommandList::~CommandList() {
|
||||
auto result = zeCommandListDestroy(_handle);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
_log.error("zeCommandListDestroy failed %#X", uint64_t(result));
|
||||
}
|
||||
}
|
||||
|
||||
CommandQueue::CommandQueue(const ze_device_handle_t& device_handle,
|
||||
const ze_context_handle_t& context,
|
||||
const ze_command_queue_priority_t& priority,
|
||||
const Config& config,
|
||||
const uint32_t& group_ordinal)
|
||||
: _context(context),
|
||||
_log("CommandQueue", config.get<LOG_LEVEL>()) {
|
||||
ze_command_queue_desc_t queue_desc =
|
||||
{ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, nullptr, group_ordinal, 0, 0, ZE_COMMAND_QUEUE_MODE_DEFAULT, priority};
|
||||
zeroUtils::throwOnFail("zeCommandQueueCreate",
|
||||
zeCommandQueueCreate(_context, device_handle, &queue_desc, &_handle));
|
||||
}
|
||||
void CommandQueue::executeCommandList(CommandList& command_list) const {
|
||||
zeroUtils::throwOnFail("zeCommandQueueExecuteCommandLists",
|
||||
zeCommandQueueExecuteCommandLists(_handle, 1, &command_list._handle, nullptr));
|
||||
}
|
||||
void CommandQueue::executeCommandList(CommandList& command_list, Fence& fence) const {
|
||||
zeroUtils::throwOnFail("zeCommandQueueExecuteCommandLists",
|
||||
zeCommandQueueExecuteCommandLists(_handle, 1, &command_list._handle, fence.handle()));
|
||||
}
|
||||
CommandQueue::~CommandQueue() {
|
||||
auto result = zeCommandQueueDestroy(_handle);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
_log.error("zeCommandQueueDestroy failed %#X", uint64_t(result));
|
||||
}
|
||||
}
|
||||
|
||||
Fence::Fence(const CommandQueue& command_queue, const Config& config) : _log("Fence", config.get<LOG_LEVEL>()) {
|
||||
ze_fence_desc_t fence_desc = {ZE_STRUCTURE_TYPE_FENCE_DESC, nullptr, 0};
|
||||
zeroUtils::throwOnFail("zeFenceCreate", zeFenceCreate(command_queue.handle(), &fence_desc, &_handle));
|
||||
}
|
||||
void Fence::reset() const {
|
||||
zeroUtils::throwOnFail("zeFenceReset", zeFenceReset(_handle));
|
||||
}
|
||||
void Fence::hostSynchronize() const {
|
||||
zeroUtils::throwOnFail("zeFenceHostSynchronize", zeFenceHostSynchronize(_handle, UINT64_MAX));
|
||||
}
|
||||
Fence::~Fence() {
|
||||
auto result = zeFenceDestroy(_handle);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
_log.error("zeFenceDestroy failed %#X", uint64_t(result));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set(TARGET_NAME "openvino_npu_driver_compiler_adapter")
|
||||
|
||||
file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h)
|
||||
source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES})
|
||||
|
||||
add_library(${TARGET_NAME} STATIC ${SOURCES})
|
||||
|
||||
ov_build_target_faster(${TARGET_NAME}
|
||||
PCH PRIVATE "src/precomp.hpp"
|
||||
)
|
||||
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
|
||||
|
||||
target_include_directories(${TARGET_NAME}
|
||||
PUBLIC
|
||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
|
||||
)
|
||||
|
||||
target_link_libraries(${TARGET_NAME}
|
||||
PUBLIC
|
||||
openvino::npu_al
|
||||
PRIVATE
|
||||
openvino_npu_zero_result_parser
|
||||
openvino::runtime
|
||||
openvino::runtime::dev
|
||||
ze_loader
|
||||
)
|
||||
|
||||
#
|
||||
# targets install
|
||||
#
|
||||
ov_install_static_lib(${TARGET_NAME} ${NPU_INTERNAL_COMPONENT})
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
#include <vector>
|
||||
|
||||
#include "npu_driver_compiler_adapter.hpp"
|
||||
#include "openvino/pass/manager.hpp"
|
||||
|
||||
/**
|
||||
* @brief Contain all required transformation on OpenVINO model in case for external compiler usage and
|
||||
* providing forward compatibility (OV model with opset N+M, external compiler with opset N)
|
||||
*/
|
||||
namespace intel_npu::driverCompilerAdapter {
|
||||
|
||||
/**
|
||||
* @brief Serialize OpenVINO model to IR
|
||||
*/
|
||||
IR serializeToIR(const std::shared_ptr<const ov::Model>& model, uint32_t supportedVersionByCompiler = 7);
|
||||
|
||||
} // namespace intel_npu::driverCompilerAdapter
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
#include "intel_npu/al/icompiler.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
namespace driverCompilerAdapter {
|
||||
|
||||
struct IR {
|
||||
std::stringstream xml;
|
||||
std::stringstream weights;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Interface for external compiler
|
||||
* @details Isolate external API calls from general logic
|
||||
*/
|
||||
class IExternalCompiler {
|
||||
public:
|
||||
virtual ~IExternalCompiler() = default;
|
||||
|
||||
/**
|
||||
* @brief Get opset supported by compiler
|
||||
*/
|
||||
virtual uint32_t getSupportedOpset() const = 0;
|
||||
|
||||
/**
|
||||
* @brief Get query result for current network
|
||||
*/
|
||||
virtual std::unordered_set<std::string> getQueryResult(IR& irModel, const Config& config) const = 0;
|
||||
|
||||
/**
|
||||
* @brief Sends the serialized model and its I/O metadata to the driver for compilation.
|
||||
* @return The compiled model descriptor corresponding to the previously given network.
|
||||
*/
|
||||
virtual NetworkDescription compileIR(const std::shared_ptr<const ov::Model>& model,
|
||||
IR& irModel,
|
||||
const Config& config) const = 0;
|
||||
virtual NetworkMetadata parseBlob(const std::vector<uint8_t>& blob, const Config& config) const = 0;
|
||||
};
|
||||
} // namespace driverCompilerAdapter
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include "iexternal_compiler.hpp"
|
||||
#include "intel_npu/al/icompiler.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
namespace driverCompilerAdapter {
|
||||
|
||||
/**
|
||||
* @brief Adapter for Compiler in driver
|
||||
* @details Wrap compiler in driver calls and do preliminary actions (like opset conversion)
|
||||
*/
|
||||
class LevelZeroCompilerAdapter final : public ICompiler {
|
||||
public:
|
||||
LevelZeroCompilerAdapter();
|
||||
|
||||
uint32_t getSupportedOpsetVersion() const override final;
|
||||
|
||||
NetworkDescription compile(const std::shared_ptr<const ov::Model>& model,
|
||||
const Config& config) const override final;
|
||||
|
||||
ov::SupportedOpsMap query(const std::shared_ptr<const ov::Model>& model, const Config& config) const override final;
|
||||
|
||||
NetworkMetadata parse(const std::vector<uint8_t>& network, const Config& config) const override final;
|
||||
|
||||
std::vector<ov::ProfilingInfo> process_profiling_output(const std::vector<uint8_t>& profData,
|
||||
const std::vector<uint8_t>& network,
|
||||
const Config& config) const override final;
|
||||
|
||||
private:
|
||||
/**
|
||||
* @brief Separate externals calls to separate class
|
||||
*/
|
||||
std::shared_ptr<IExternalCompiler> apiAdapter;
|
||||
ze_driver_handle_t _driverHandle = nullptr;
|
||||
mutable Logger _logger;
|
||||
};
|
||||
|
||||
} // namespace driverCompilerAdapter
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,220 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include <ze_api.h>
|
||||
#include <ze_graph_ext.h>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "iexternal_compiler.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
namespace driverCompilerAdapter {
|
||||
|
||||
#define NotSupportLogHandle(T) \
|
||||
(std::is_same<T, ze_graph_dditable_ext_t>::value || std::is_same<T, ze_graph_dditable_ext_1_1_t>::value || \
|
||||
std::is_same<T, ze_graph_dditable_ext_1_2_t>::value || std::is_same<T, ze_graph_dditable_ext_1_3_t>::value)
|
||||
|
||||
#define NotSupportQuery(T) \
|
||||
(std::is_same<T, ze_graph_dditable_ext_t>::value || std::is_same<T, ze_graph_dditable_ext_1_1_t>::value || \
|
||||
std::is_same<T, ze_graph_dditable_ext_1_2_t>::value)
|
||||
|
||||
// ext version == 1.3 && 1.4, support API (pfnQueryNetworkCreate, pfnQueryNetworkDestroy,
|
||||
// pfnQueryNetworkGetSupportedLayers)
|
||||
#define SupportAPIGraphQueryNetworkV1(T) \
|
||||
(std::is_same<T, ze_graph_dditable_ext_1_3_t>::value || std::is_same<T, ze_graph_dditable_ext_1_4_t>::value)
|
||||
|
||||
// ext version >= 1.5, support API (pfnCreate2, pfnQueryNetworkCreate2, pfnQueryContextMemory)
|
||||
#define SupportAPIGraphQueryNetworkV2(T) ((!NotSupportQuery(T) && !SupportAPIGraphQueryNetworkV1(T)))
|
||||
|
||||
// For ext version >= 1.5, pfnCreate2 api is avaible
|
||||
#define NotSupportGraph2(T) \
|
||||
(std::is_same<T, ze_graph_dditable_ext_t>::value || std::is_same<T, ze_graph_dditable_ext_1_1_t>::value || \
|
||||
std::is_same<T, ze_graph_dditable_ext_1_2_t>::value || std::is_same<T, ze_graph_dditable_ext_1_3_t>::value || \
|
||||
std::is_same<T, ze_graph_dditable_ext_1_4_t>::value)
|
||||
|
||||
/**
|
||||
* Adapter to use CiD through ZeroAPI
|
||||
*/
|
||||
template <typename TableExtension>
|
||||
class LevelZeroCompilerInDriver final : public IExternalCompiler {
|
||||
public:
|
||||
LevelZeroCompilerInDriver(const char* extName, ze_driver_handle_t driverHandle);
|
||||
LevelZeroCompilerInDriver(const LevelZeroCompilerInDriver&) = delete;
|
||||
LevelZeroCompilerInDriver& operator=(const LevelZeroCompilerInDriver&) = delete;
|
||||
~LevelZeroCompilerInDriver() override;
|
||||
|
||||
uint32_t getSupportedOpset() const override;
|
||||
|
||||
std::unordered_set<std::string> getQueryResult(IR& irModel, const Config& config) const override;
|
||||
|
||||
NetworkDescription compileIR(const std::shared_ptr<const ov::Model>& model,
|
||||
IR& irModel,
|
||||
const Config& config) const override final;
|
||||
|
||||
NetworkMetadata parseBlob(const std::vector<uint8_t>& blob, const Config& config) const override final;
|
||||
|
||||
template <typename T = TableExtension, std::enable_if_t<!NotSupportQuery(T), bool> = true>
|
||||
std::unordered_set<std::string> getQueryResultFromSupportedLayers(
|
||||
ze_result_t result,
|
||||
ze_graph_query_network_handle_t& hGraphQueryNetwork) const;
|
||||
|
||||
std::vector<uint8_t> getSerializedIR(std::string& buildFlags, IR& irModel, const Config& config) const;
|
||||
|
||||
/**
|
||||
* @brief Serialize input / output information to string format.
|
||||
* @details Format:
|
||||
* --inputs_precisions="<input1Name>:<input1Precision> [<input2Name>:<input2Precision>]"
|
||||
* --inputs_layouts="<input1Name>:<input1Layout> [<input2Name>:<input2Layout>]"
|
||||
* --outputs_precisions="<output1Name>:<output1Precision>"
|
||||
* --outputs_layouts="<output1Name>:<output1Layout>"
|
||||
*
|
||||
* Since the layout information is no longer an important part of the metadata values when using the 2.0 OV API, the
|
||||
* layout fields shall be filled with default values in order to assure the backward compatibility with the driver.
|
||||
*/
|
||||
static std::string serializeIOInfo(const std::shared_ptr<const ov::Model>& model);
|
||||
|
||||
private:
|
||||
NetworkMetadata getNetworkMeta(ze_graph_handle_t graphHandle) const;
|
||||
|
||||
std::vector<uint8_t> serializeIR(IR& irModel, ze_graph_compiler_version_info_t compilerVersion) const;
|
||||
std::string serializeConfig(const Config& config, ze_graph_compiler_version_info_t& compilerVersion) const;
|
||||
|
||||
/**
|
||||
* @brief Extracts the layout value or the state descriptor from the given Level Zero structure.
|
||||
* @details Extracting the layout information is required only when using older driver versions which rely on this
|
||||
* legacy attribute. Since this information is not found within the parameter/result nodes, we need to extract this
|
||||
* value here.
|
||||
*
|
||||
* The state variables are also not found in the previously mentioned nodes, thus if the given Level Zero parameter
|
||||
* corresponds to an input/output, we shall extract the layout value from it. Else it represents a state variable
|
||||
* and the descriptor will be extracted and stored in an OpenVINO specific format.
|
||||
* @param parameters Holds the already extracted input node descriptors. The transposed shape attribute of the
|
||||
* corresponding entry may be updated according to the extracted layout value.
|
||||
* @param results Holds the already extracted output node descriptors. The transposed shape attribute of the
|
||||
* corresponding entry may be updated according to the extracted layout value.
|
||||
* @param states The state descriptors shall be stored here in an OpenVINO specific format.
|
||||
* @param stateNames The output location of the state variables' names in the order found within the compiled model.
|
||||
* @param arg The Level Zero specific structure from which the layout value or state variable descriptor shall be
|
||||
* extracted.
|
||||
*/
|
||||
template <typename T>
|
||||
void getLayoutOrStateDescriptor(IONodeDescriptorMap& parameters,
|
||||
IONodeDescriptorMap& results,
|
||||
IONodeDescriptorMap& states,
|
||||
std::vector<std::string>& stateNames,
|
||||
const T& arg) const;
|
||||
|
||||
/**
|
||||
* @brief Extracts the node or state descriptor from the given Level Zero structure.
|
||||
* @details The first version of the "ze_graph_dditable" extension does not directly support extracting the
|
||||
* information corresponding to the parameter/result nodes. However, the same information can be extracted from the
|
||||
* legacy I/O structures, with the exception of the node and tensor names fields.
|
||||
*
|
||||
* Thus, the current function is a best effort method of assuring the backward compatibility with the driver.
|
||||
* The missing names have been replaced with the only one available in the structure (i.e. "arg.name"), this
|
||||
* could potentially lead to issues further in the application's flow.
|
||||
* @param parameters The parameter node descriptors shall be stored here in an OpenVINO specific format.
|
||||
* @param results The result node descriptors shall be stored here in an OpenVINO specific format.
|
||||
* @param states The state descriptors shall be stored here in an OpenVINO specific format.
|
||||
* @param inputNames The output location of the input names in the order found within the compiled model.
|
||||
* @param outputNames The output location of the output names in the order found within the compiled model.
|
||||
* @param stateNames The output location of the state variables' names in the order found within the compiled model.
|
||||
* @param arg The Level Zero specific structure from which the node or state variable descriptor shall be
|
||||
* extracted.
|
||||
*/
|
||||
template <typename T = TableExtension,
|
||||
std::enable_if_t<std::is_same<T, ze_graph_dditable_ext_t>::value, bool> = true>
|
||||
void getNodeOrStateDescriptorLegacy(IONodeDescriptorMap& parameters,
|
||||
IONodeDescriptorMap& results,
|
||||
IONodeDescriptorMap& states,
|
||||
std::vector<std::string>& inputNames,
|
||||
std::vector<std::string>& outputNames,
|
||||
std::vector<std::string>& stateNames,
|
||||
const ze_graph_argument_properties_t& arg) const;
|
||||
|
||||
void getMetadata(TableExtension* graphDdiTableExt,
|
||||
ze_graph_handle_t graphHandle,
|
||||
uint32_t index,
|
||||
std::vector<std::string>& inputNames,
|
||||
std::vector<std::string>& outputNames,
|
||||
std::vector<std::string>& stateNames,
|
||||
IONodeDescriptorMap& parameters,
|
||||
IONodeDescriptorMap& results,
|
||||
IONodeDescriptorMap& state) const;
|
||||
|
||||
// ext version >= 1.5, support API (pfnCreate2, pfnQueryNetworkCreate2, pfnQueryContextMemory)
|
||||
template <typename T = TableExtension, typename std::enable_if_t<SupportAPIGraphQueryNetworkV2(T), bool> = true>
|
||||
std::unordered_set<std::string> queryImpl(IR& irModel, const Config& config) const;
|
||||
|
||||
// ext version == 1.3 && 1.4, support API (pfnQueryNetworkCreate, pfnQueryNetworkDestroy,
|
||||
// pfnQueryNetworkGetSupportedLayers)
|
||||
template <typename T = TableExtension, typename std::enable_if_t<SupportAPIGraphQueryNetworkV1(T), bool> = true>
|
||||
std::unordered_set<std::string> queryImpl(IR& irModel, const Config& config) const;
|
||||
|
||||
// For ext version < 1.3
|
||||
template <typename T = TableExtension, typename std::enable_if_t<NotSupportQuery(T), bool> = true>
|
||||
std::unordered_set<std::string> queryImpl(IR& irModel, const Config& config) const;
|
||||
|
||||
template <typename T = TableExtension, typename std::enable_if_t<NotSupportGraph2(T), bool> = true>
|
||||
ze_result_t createGraph(const ze_graph_format_t& format,
|
||||
const std::vector<uint8_t>& serializedIR,
|
||||
const std::string& buildFlags,
|
||||
const uint32_t& flags,
|
||||
ze_graph_handle_t* graph) const;
|
||||
|
||||
template <typename T = TableExtension, typename std::enable_if_t<!NotSupportGraph2(T), bool> = true>
|
||||
ze_result_t createGraph(const ze_graph_format_t& format,
|
||||
const std::vector<uint8_t>& serializedIR,
|
||||
const std::string& buildFlags,
|
||||
const uint32_t& flags,
|
||||
ze_graph_handle_t* graph) const;
|
||||
|
||||
template <typename T = TableExtension, typename std::enable_if_t<!NotSupportLogHandle(T), bool> = true>
|
||||
std::string getLatestBuildError() const;
|
||||
|
||||
template <typename T = TableExtension, typename std::enable_if_t<NotSupportLogHandle(T), bool> = true>
|
||||
std::string getLatestBuildError() const {
|
||||
return "";
|
||||
}
|
||||
|
||||
private:
|
||||
ze_driver_handle_t _driverHandle = nullptr;
|
||||
ze_device_handle_t _deviceHandle = nullptr;
|
||||
ze_context_handle_t _context = nullptr;
|
||||
|
||||
TableExtension* _graphDdiTableExt = nullptr;
|
||||
mutable Logger _logger;
|
||||
};
|
||||
|
||||
template <typename TableExtension>
|
||||
LevelZeroCompilerInDriver<TableExtension>::LevelZeroCompilerInDriver(const char* extName,
|
||||
ze_driver_handle_t driverHandle)
|
||||
: _driverHandle(driverHandle),
|
||||
_logger("LevelZeroCompilerInDriver", Logger::global().level()) {
|
||||
// Load our graph extension
|
||||
auto result =
|
||||
zeDriverGetExtensionFunctionAddress(_driverHandle, extName, reinterpret_cast<void**>(&_graphDdiTableExt));
|
||||
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("Failed to initialize zeDriver. Error code: ", std::hex, result);
|
||||
}
|
||||
|
||||
uint32_t deviceCount = 1;
|
||||
// Get our target device
|
||||
result = zeDeviceGet(_driverHandle, &deviceCount, &_deviceHandle);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("Failed to get device. Error code: ", std::hex, result);
|
||||
}
|
||||
|
||||
ze_context_desc_t contextDesc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
|
||||
result = zeContextCreate(_driverHandle, &contextDesc, &_context);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("Failed to initialize context for device. Error code: ", std::hex, result);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace driverCompilerAdapter
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "graph_transformations.hpp"
|
||||
|
||||
#include <istream>
|
||||
#include <mutex>
|
||||
|
||||
#include "openvino/pass/serialize.hpp"
|
||||
#include "transformations/op_conversions/convert_interpolate11_downgrade.hpp"
|
||||
|
||||
namespace intel_npu::driverCompilerAdapter {
|
||||
|
||||
IR serializeToIR(const std::shared_ptr<const ov::Model>& origModel, uint32_t supportedOpset) {
|
||||
// There is no const variant of run_passes so use const_cast here
|
||||
// as model serialization does not mutate the model
|
||||
std::shared_ptr<ov::Model> model = std::const_pointer_cast<ov::Model>(origModel);
|
||||
|
||||
const auto passConfig = std::make_shared<ov::pass::PassConfig>();
|
||||
ov::pass::Manager manager(passConfig);
|
||||
|
||||
if (supportedOpset < 11) {
|
||||
// Need to clone to modify the model and remain thread safe
|
||||
model = model->clone();
|
||||
// Downgrade to opset10
|
||||
manager.register_pass<ov::pass::ConvertInterpolate11ToInterpolate4>();
|
||||
}
|
||||
|
||||
std::stringstream xmlStream, weightsStream;
|
||||
manager.register_pass<ov::pass::Serialize>(xmlStream, weightsStream);
|
||||
|
||||
// Depending on the driver version, the compiler attached to it may request this information as an indicator of the
|
||||
// precision/layout preprocessing requirement. We are setting this value to "true" since the API version is no
|
||||
// longer a cause for altering the metadata. This is due to the preprocessing performed in the OpenVINO framework's
|
||||
// implementaion, the "ov::Model" object is preprocessed before reaching the NPU plugin.
|
||||
const auto new_api_key = "is_new_api";
|
||||
|
||||
// We modify the original model object here therefore a mutex is required
|
||||
static std::mutex rtInfoMutex;
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(rtInfoMutex);
|
||||
|
||||
model->set_rt_info(true, new_api_key);
|
||||
|
||||
manager.run_passes(model);
|
||||
|
||||
auto& rtInfo = model->get_rt_info();
|
||||
rtInfo.erase(new_api_key);
|
||||
}
|
||||
|
||||
return {std::move(xmlStream), std::move(weightsStream)};
|
||||
}
|
||||
|
||||
} // namespace intel_npu::driverCompilerAdapter
|
||||
|
|
@ -0,0 +1,233 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "npu_driver_compiler_adapter.hpp"
|
||||
|
||||
#include "graph_transformations.hpp"
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
#include "intel_npu/utils/zero/zero_result.hpp"
|
||||
#include "ze_intel_vpu_uuid.h"
|
||||
#include "zero_compiler_in_driver.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
namespace driverCompilerAdapter {
|
||||
|
||||
LevelZeroCompilerAdapter::LevelZeroCompilerAdapter() : _logger("LevelZeroCompilerAdapter", Logger::global().level()) {
|
||||
_logger.debug("initialize zeAPI");
|
||||
auto result = zeInit(ZE_INIT_FLAG_VPU_ONLY);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("L0 initialize zeAPI",
|
||||
" result: ",
|
||||
ze_result_to_string(result),
|
||||
", code 0x",
|
||||
std::hex,
|
||||
uint64_t(result));
|
||||
}
|
||||
uint32_t drivers = 0;
|
||||
result = zeDriverGet(&drivers, nullptr);
|
||||
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("L0 zeDriverGet get count",
|
||||
" result: ",
|
||||
ze_result_to_string(result),
|
||||
", code 0x",
|
||||
std::hex,
|
||||
uint64_t(result));
|
||||
}
|
||||
|
||||
std::vector<ze_driver_handle_t> allDrivers(drivers);
|
||||
result = zeDriverGet(&drivers, allDrivers.data());
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("L0 zeDriverGet get drivers",
|
||||
" result: ",
|
||||
ze_result_to_string(result),
|
||||
", code 0x",
|
||||
std::hex,
|
||||
uint64_t(result));
|
||||
}
|
||||
|
||||
const ze_driver_uuid_t uuid = ze_intel_vpu_driver_uuid;
|
||||
ze_driver_properties_t props = {};
|
||||
props.stype = ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES;
|
||||
// Get our target driver
|
||||
for (uint32_t i = 0; i < drivers; ++i) {
|
||||
result = zeDriverGetProperties(allDrivers[i], &props);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("L0 zeDriverGetProperties",
|
||||
" result: ",
|
||||
ze_result_to_string(result),
|
||||
", code 0x",
|
||||
std::hex,
|
||||
uint64_t(result));
|
||||
}
|
||||
if (memcmp(&props.uuid, &uuid, sizeof(uuid)) == 0) {
|
||||
_driverHandle = allDrivers[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (_driverHandle == nullptr) {
|
||||
OPENVINO_THROW("LevelZeroCompilerAdapter: Failed to get properties about zeDriver");
|
||||
return;
|
||||
}
|
||||
|
||||
// query the extension properties
|
||||
uint32_t count = 0;
|
||||
result = zeDriverGetExtensionProperties(_driverHandle, &count, nullptr);
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("L0 zeDriverGetExtensionProperties get count",
|
||||
" result: ",
|
||||
ze_result_to_string(result),
|
||||
", code 0x",
|
||||
std::hex,
|
||||
uint64_t(result));
|
||||
}
|
||||
std::vector<ze_driver_extension_properties_t> extProps;
|
||||
extProps.resize(count);
|
||||
result = zeDriverGetExtensionProperties(_driverHandle, &count, extProps.data());
|
||||
if (ZE_RESULT_SUCCESS != result) {
|
||||
OPENVINO_THROW("L0 zeDriverGetExtensionProperties get properties",
|
||||
" result: ",
|
||||
ze_result_to_string(result),
|
||||
", code 0x",
|
||||
std::hex,
|
||||
uint64_t(result));
|
||||
}
|
||||
const char* graphExtName = nullptr;
|
||||
uint32_t targetVersion = 0;
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
auto& property = extProps[i];
|
||||
|
||||
if (strncmp(property.name, ZE_GRAPH_EXT_NAME, strlen(ZE_GRAPH_EXT_NAME)) != 0)
|
||||
continue;
|
||||
|
||||
// If the driver version is latest, will just use its name.
|
||||
if (property.version == ZE_GRAPH_EXT_VERSION_CURRENT) {
|
||||
graphExtName = property.name;
|
||||
targetVersion = property.version;
|
||||
break;
|
||||
}
|
||||
|
||||
// Use the latest version supported by the driver.
|
||||
if (property.version > targetVersion) {
|
||||
graphExtName = property.name;
|
||||
targetVersion = property.version;
|
||||
}
|
||||
}
|
||||
|
||||
if (graphExtName == nullptr) {
|
||||
OPENVINO_THROW("LevelZeroCompilerAdapter: Failed to find Graph extension in NPU Driver");
|
||||
}
|
||||
|
||||
const uint16_t adapterMajorVersion = 1;
|
||||
uint16_t driverMajorVersion = ZE_MAJOR_VERSION(targetVersion);
|
||||
if (adapterMajorVersion != driverMajorVersion) {
|
||||
OPENVINO_THROW("LevelZeroCompilerAdapter: adapterMajorVersion: ",
|
||||
adapterMajorVersion,
|
||||
" and driverMajorVersion: ",
|
||||
driverMajorVersion,
|
||||
" mismatch!");
|
||||
}
|
||||
|
||||
#if defined(NPU_PLUGIN_DEVELOPER_BUILD)
|
||||
auto adapterManualConfig = std::getenv("ADAPTER_MANUAL_CONFIG");
|
||||
if (adapterManualConfig != nullptr) {
|
||||
if (strcmp(adapterManualConfig, "ZE_extension_graph_1_5") == 0) {
|
||||
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_5");
|
||||
targetVersion = ZE_GRAPH_EXT_VERSION_1_5;
|
||||
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_4") == 0) {
|
||||
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_4");
|
||||
targetVersion = ZE_GRAPH_EXT_VERSION_1_4;
|
||||
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_3") == 0) {
|
||||
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_3");
|
||||
targetVersion = ZE_GRAPH_EXT_VERSION_1_3;
|
||||
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_2") == 0) {
|
||||
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_2");
|
||||
targetVersion = ZE_GRAPH_EXT_VERSION_1_2;
|
||||
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_1") == 0) {
|
||||
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_1");
|
||||
targetVersion = ZE_GRAPH_EXT_VERSION_1_1;
|
||||
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_0") == 0) {
|
||||
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_0");
|
||||
targetVersion = ZE_GRAPH_EXT_VERSION_1_0;
|
||||
} else {
|
||||
OPENVINO_THROW("Using unsupported ADAPTER_MANUAL_CONFIG!");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (ZE_GRAPH_EXT_VERSION_1_1 == targetVersion) {
|
||||
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_1");
|
||||
apiAdapter =
|
||||
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_1_t>>(graphExtName, _driverHandle);
|
||||
} else if (ZE_GRAPH_EXT_VERSION_1_2 == targetVersion) {
|
||||
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_2");
|
||||
apiAdapter =
|
||||
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_2_t>>(graphExtName, _driverHandle);
|
||||
} else if (strcmp(graphExtName, ZE_GRAPH_EXT_NAME_1_3) == 0) {
|
||||
apiAdapter =
|
||||
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_3_t>>(graphExtName, _driverHandle);
|
||||
} else if (ZE_GRAPH_EXT_VERSION_1_4 == targetVersion) {
|
||||
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_4");
|
||||
apiAdapter =
|
||||
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_4_t>>(graphExtName, _driverHandle);
|
||||
} else if (ZE_GRAPH_EXT_VERSION_1_5 == targetVersion) {
|
||||
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_5");
|
||||
apiAdapter =
|
||||
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_5_t>>(graphExtName, _driverHandle);
|
||||
} else {
|
||||
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_0");
|
||||
apiAdapter = std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_t>>(graphExtName, _driverHandle);
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t LevelZeroCompilerAdapter::getSupportedOpsetVersion() const {
|
||||
return apiAdapter->getSupportedOpset();
|
||||
}
|
||||
|
||||
NetworkDescription LevelZeroCompilerAdapter::compile(const std::shared_ptr<const ov::Model>& model,
|
||||
const Config& config) const {
|
||||
_logger.setLevel(config.get<LOG_LEVEL>());
|
||||
_logger.debug("compileIR");
|
||||
uint32_t supportedOpset = apiAdapter->getSupportedOpset();
|
||||
|
||||
auto IR = serializeToIR(model, supportedOpset);
|
||||
|
||||
return apiAdapter->compileIR(model, IR, config);
|
||||
}
|
||||
|
||||
ov::SupportedOpsMap LevelZeroCompilerAdapter::query(const std::shared_ptr<const ov::Model>& model,
|
||||
const Config& config) const {
|
||||
_logger.setLevel(config.get<LOG_LEVEL>());
|
||||
_logger.debug("queryResult");
|
||||
ov::SupportedOpsMap result;
|
||||
const std::string deviceName = "NPU";
|
||||
|
||||
auto IR = serializeToIR(model);
|
||||
try {
|
||||
const auto supportedLayers = apiAdapter->getQueryResult(IR, config);
|
||||
for (auto&& layerName : supportedLayers) {
|
||||
result.emplace(layerName, deviceName);
|
||||
}
|
||||
_logger.info("For given model, there are %d supported layers", supportedLayers.size());
|
||||
} catch (std::exception& e) {
|
||||
OPENVINO_THROW("Fail in calling querynetwork : ", e.what());
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
NetworkMetadata LevelZeroCompilerAdapter::parse(const std::vector<uint8_t>& blob, const Config& config) const {
|
||||
_logger.setLevel(config.get<LOG_LEVEL>());
|
||||
_logger.debug("parseBlob");
|
||||
return apiAdapter->parseBlob(blob, config);
|
||||
}
|
||||
|
||||
std::vector<ov::ProfilingInfo> LevelZeroCompilerAdapter::process_profiling_output(const std::vector<uint8_t>&,
|
||||
const std::vector<uint8_t>&,
|
||||
const Config&) const {
|
||||
OPENVINO_THROW("Profiling post-processing is not implemented.");
|
||||
}
|
||||
|
||||
} // namespace driverCompilerAdapter
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cassert>
|
||||
#include <cctype>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <functional>
|
||||
#include <initializer_list>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
#include <tuple>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,66 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
file(GLOB SOURCES "src/*.cpp" "include/*.hpp")
|
||||
|
||||
set(NPU_PLUGIN_TARGET "openvino_intel_npu_plugin")
|
||||
set(NPU_PLUGIN_ENGINE_SOURCE_FILE "src/plugin.cpp")
|
||||
set(TARGET_NAME ${NPU_PLUGIN_TARGET})
|
||||
|
||||
#
|
||||
# Plugin library
|
||||
#
|
||||
|
||||
ov_add_plugin(NAME ${NPU_PLUGIN_TARGET}
|
||||
DEVICE_NAME ${NPU_DEVICE_NAME}
|
||||
SOURCES ${SOURCES}
|
||||
VERSION_DEFINES_FOR ${NPU_PLUGIN_ENGINE_SOURCE_FILE}
|
||||
)
|
||||
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
|
||||
|
||||
if(ENABLE_DRIVER_COMPILER_ADAPTER)
|
||||
target_compile_definitions(${TARGET_NAME} PRIVATE ENABLE_DRIVER_COMPILER_ADAPTER)
|
||||
target_link_libraries(${TARGET_NAME}
|
||||
PRIVATE
|
||||
openvino_npu_driver_compiler_adapter
|
||||
)
|
||||
endif()
|
||||
|
||||
if(ENABLE_ZEROAPI_BACKEND)
|
||||
target_compile_definitions(${TARGET_NAME} PRIVATE ENABLE_ZEROAPI_BACKEND)
|
||||
target_link_libraries(${TARGET_NAME}
|
||||
PRIVATE
|
||||
openvino_npu_level_zero_backend
|
||||
)
|
||||
endif()
|
||||
|
||||
if(ENABLE_IMD_BACKEND)
|
||||
target_compile_definitions(${TARGET_NAME} PRIVATE ENABLE_IMD_BACKEND)
|
||||
endif()
|
||||
|
||||
if(NOT BUILD_SHARED_LIBS)
|
||||
target_compile_definitions(${TARGET_NAME} PRIVATE OPENVINO_STATIC_LIBRARY)
|
||||
endif()
|
||||
|
||||
target_link_libraries(${TARGET_NAME}
|
||||
PRIVATE
|
||||
openvino::npu_al
|
||||
openvino::npu_logger_utils
|
||||
openvino::runtime
|
||||
openvino::runtime::dev
|
||||
)
|
||||
|
||||
target_include_directories(${TARGET_NAME}
|
||||
PRIVATE
|
||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
|
||||
)
|
||||
target_include_directories(${TARGET_NAME}
|
||||
SYSTEM PRIVATE
|
||||
${NPU_PLUGIN_SOURCE_DIR}/thirdparty/level-zero/include
|
||||
${NPU_PLUGIN_SOURCE_DIR}/thirdparty/level-zero-ext
|
||||
)
|
||||
|
||||
ov_add_api_validator_post_build_step(TARGET ${NPU_PLUGIN_TARGET})
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "openvino/runtime/iasync_infer_request.hpp"
|
||||
#include "sync_infer_request.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
class AsyncInferRequest final : public ov::IAsyncInferRequest {
|
||||
public:
|
||||
explicit AsyncInferRequest(const std::shared_ptr<SyncInferRequest>& inferRequest,
|
||||
const std::shared_ptr<ov::threading::ITaskExecutor>& requestExecutor,
|
||||
const std::shared_ptr<ov::threading::ITaskExecutor>& getResultExecutor,
|
||||
const std::shared_ptr<ov::threading::ITaskExecutor>& callbackExecutor);
|
||||
|
||||
AsyncInferRequest(const AsyncInferRequest&) = delete;
|
||||
|
||||
AsyncInferRequest& operator=(const AsyncInferRequest&) = delete;
|
||||
|
||||
~AsyncInferRequest();
|
||||
|
||||
std::shared_ptr<SyncInferRequest> get_sync_infer_request() {
|
||||
return _syncInferRequest;
|
||||
}
|
||||
|
||||
private:
|
||||
std::shared_ptr<SyncInferRequest> _syncInferRequest;
|
||||
std::shared_ptr<ov::threading::ITaskExecutor> _getResultExecutor;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "intel_npu/al/icompiled_model.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "npu.hpp"
|
||||
#include "openvino/runtime/so_ptr.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
class CompiledModel final : public ICompiledModel {
|
||||
public:
|
||||
explicit CompiledModel(const std::shared_ptr<const ov::Model>& model,
|
||||
const std::shared_ptr<const ov::IPlugin>& plugin,
|
||||
const std::shared_ptr<const NetworkDescription>& networkDescription,
|
||||
const std::shared_ptr<IDevice>& device,
|
||||
const std::optional<ov::SoPtr<ICompiler>>& compiler,
|
||||
const Config& config);
|
||||
|
||||
CompiledModel(const CompiledModel&) = delete;
|
||||
|
||||
CompiledModel& operator=(const CompiledModel&) = delete;
|
||||
|
||||
std::shared_ptr<ov::IAsyncInferRequest> create_infer_request() const override;
|
||||
|
||||
std::shared_ptr<ov::ISyncInferRequest> create_sync_infer_request() const override;
|
||||
|
||||
void export_model(std::ostream& stream) const override;
|
||||
|
||||
std::shared_ptr<const ov::Model> get_runtime_model() const override;
|
||||
|
||||
void set_property(const ov::AnyMap& properties) override;
|
||||
|
||||
ov::Any get_property(const std::string& name) const override;
|
||||
|
||||
const std::shared_ptr<const NetworkDescription>& get_network_description() const override;
|
||||
|
||||
const Config& get_config() const override;
|
||||
|
||||
const ov::SoPtr<ICompiler>& get_compiler() const override;
|
||||
|
||||
private:
|
||||
void initialize_properties();
|
||||
|
||||
void configure_stream_executors();
|
||||
|
||||
std::shared_ptr<const NetworkDescription> _networkPtr;
|
||||
const std::shared_ptr<const ov::Model> _model;
|
||||
const Config _config;
|
||||
Logger _logger;
|
||||
const std::shared_ptr<IDevice> _device;
|
||||
mutable std::shared_ptr<IExecutor> _executorPtr;
|
||||
std::shared_ptr<ov::threading::ITaskExecutor> _resultExecutor;
|
||||
|
||||
// properties map: {name -> [supported, mutable, eval function]}
|
||||
std::map<std::string, std::tuple<bool, ov::PropertyMutability, std::function<ov::Any(const Config&)>>> _properties;
|
||||
std::vector<ov::PropertyName> _supportedProperties;
|
||||
|
||||
std::optional<ov::SoPtr<ICompiler>> _compiler;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
// System
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
#include "openvino/runtime/so_ptr.hpp"
|
||||
|
||||
// Plugin
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "npu.hpp"
|
||||
#include "npu_private_properties.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
enum class AvailableBackends { LEVEL_ZERO, IMD };
|
||||
|
||||
/** @brief Represent container for all backends and hide all related searching logic */
|
||||
class NPUBackends final {
|
||||
public:
|
||||
explicit NPUBackends(const std::vector<AvailableBackends>& backendRegistry, const Config& config);
|
||||
|
||||
std::shared_ptr<IDevice> getDevice(const std::string& specificName = "") const;
|
||||
std::shared_ptr<IDevice> getDevice(const ov::AnyMap& paramMap) const;
|
||||
std::vector<std::string> getAvailableDevicesNames() const;
|
||||
std::string getBackendName() const;
|
||||
void registerOptions(OptionsDesc& options) const;
|
||||
std::string getCompilationPlatform(const std::string_view platform, const std::string& deviceId) const;
|
||||
|
||||
void setup(const Config& config);
|
||||
|
||||
private:
|
||||
Logger _logger;
|
||||
ov::SoPtr<IEngineBackend> _backend;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
// Compiler Factory
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "intel_npu/al/icompiler.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "npu_private_properties.hpp" // AL
|
||||
#include "openvino/runtime/so_ptr.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
ov::SoPtr<ICompiler> createCompiler(ov::intel_npu::CompilerType compilerType, const Logger& log);
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
#include "npu.hpp"
|
||||
#include "npu_backends.hpp"
|
||||
#include "npu_private_properties.hpp"
|
||||
#include "openvino/runtime/intel_npu/properties.hpp"
|
||||
#include "openvino/runtime/internal_properties.hpp"
|
||||
#include "openvino/runtime/properties.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
class Metrics final {
|
||||
public:
|
||||
Metrics(const std::shared_ptr<const NPUBackends>& backends);
|
||||
|
||||
std::vector<std::string> GetAvailableDevicesNames() const;
|
||||
const std::vector<std::string>& SupportedMetrics() const;
|
||||
std::string GetFullDeviceName(const std::string& specifiedDeviceName) const;
|
||||
IDevice::Uuid GetDeviceUuid(const std::string& specifiedDeviceName) const;
|
||||
const std::vector<std::string>& GetSupportedConfigKeys() const;
|
||||
const std::vector<std::string>& GetOptimizationCapabilities() const;
|
||||
const std::tuple<uint32_t, uint32_t, uint32_t>& GetRangeForAsyncInferRequest() const;
|
||||
const std::tuple<uint32_t, uint32_t>& GetRangeForStreams() const;
|
||||
std::string GetDeviceArchitecture(const std::string& specifiedDeviceName) const;
|
||||
std::string GetBackendName() const;
|
||||
uint64_t GetDeviceAllocMemSize(const std::string& specifiedDeviceName) const;
|
||||
uint64_t GetDeviceTotalMemSize(const std::string& specifiedDeviceName) const;
|
||||
uint32_t GetDriverVersion(const std::string& specifiedDeviceName) const;
|
||||
|
||||
std::vector<ov::PropertyName> GetCachingProperties() const;
|
||||
std::vector<ov::PropertyName> GetInternalSupportedProperties() const;
|
||||
|
||||
~Metrics() = default;
|
||||
|
||||
private:
|
||||
const std::shared_ptr<const NPUBackends> _backends;
|
||||
std::vector<std::string> _supportedMetrics;
|
||||
std::vector<std::string> _supportedConfigKeys;
|
||||
const std::vector<std::string> _optimizationCapabilities = {
|
||||
ov::device::capability::FP16,
|
||||
ov::device::capability::INT8,
|
||||
ov::device::capability::EXPORT_IMPORT,
|
||||
};
|
||||
const std::vector<ov::PropertyName> _cachingProperties = {ov::device::architecture.name(),
|
||||
ov::intel_npu::compilation_mode_params.name(),
|
||||
ov::intel_npu::tiles.name(),
|
||||
ov::intel_npu::dpu_groups.name(),
|
||||
ov::intel_npu::dma_engines.name(),
|
||||
ov::intel_npu::compilation_mode.name(),
|
||||
ov::intel_npu::driver_version.name(),
|
||||
ov::intel_npu::compiler_type.name(),
|
||||
ov::intel_npu::use_elf_compiler_backend.name()};
|
||||
|
||||
const std::vector<ov::PropertyName> _internalSupportedProperties = {ov::internal::caching_properties.name()};
|
||||
|
||||
// Metric to provide a hint for a range for number of async infer requests. (bottom bound, upper bound, step)
|
||||
const std::tuple<uint32_t, uint32_t, uint32_t> _rangeForAsyncInferRequests{1u, 10u, 1u};
|
||||
|
||||
// Metric to provide information about a range for streams.(bottom bound, upper bound)
|
||||
const std::tuple<uint32_t, uint32_t> _rangeForStreams{1u, 4u};
|
||||
|
||||
std::string getDeviceName(const std::string& specifiedDeviceName) const;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "intel_npu/al/config/config.hpp"
|
||||
#include "intel_npu/al/icompiler.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "npu.hpp"
|
||||
#include "npu_backends.hpp"
|
||||
#include "npu_metrics.hpp"
|
||||
#include "openvino/runtime/iplugin.hpp"
|
||||
#include "openvino/runtime/so_ptr.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
class Plugin : public ov::IPlugin {
|
||||
public:
|
||||
Plugin();
|
||||
|
||||
Plugin(const Plugin&) = delete;
|
||||
|
||||
Plugin& operator=(const Plugin&) = delete;
|
||||
|
||||
virtual ~Plugin() = default;
|
||||
|
||||
void set_property(const ov::AnyMap& properties) override;
|
||||
|
||||
ov::Any get_property(const std::string& name, const ov::AnyMap& arguments) const override;
|
||||
|
||||
std::shared_ptr<ov::ICompiledModel> compile_model(const std::shared_ptr<const ov::Model>& model,
|
||||
const ov::AnyMap& properties) const override;
|
||||
|
||||
std::shared_ptr<ov::ICompiledModel> compile_model(const std::shared_ptr<const ov::Model>& model,
|
||||
const ov::AnyMap& properties,
|
||||
const ov::SoPtr<ov::IRemoteContext>& context) const override;
|
||||
|
||||
ov::SoPtr<ov::IRemoteContext> create_context(const ov::AnyMap& remote_properties) const override;
|
||||
|
||||
ov::SoPtr<ov::IRemoteContext> get_default_context(const ov::AnyMap& remote_properties) const override;
|
||||
|
||||
std::shared_ptr<ov::ICompiledModel> import_model(std::istream& stream, const ov::AnyMap& properties) const override;
|
||||
|
||||
std::shared_ptr<ov::ICompiledModel> import_model(std::istream& stream,
|
||||
const ov::SoPtr<ov::IRemoteContext>& context,
|
||||
const ov::AnyMap& properties) const override;
|
||||
|
||||
ov::SupportedOpsMap query_model(const std::shared_ptr<const ov::Model>& model,
|
||||
const ov::AnyMap& properties) const override;
|
||||
|
||||
private:
|
||||
ov::SoPtr<ICompiler> getCompiler(const Config& config) const;
|
||||
|
||||
std::shared_ptr<NPUBackends> _backends;
|
||||
|
||||
std::map<std::string, std::string> _config;
|
||||
std::shared_ptr<OptionsDesc> _options;
|
||||
Config _globalConfig;
|
||||
Logger _logger;
|
||||
std::unique_ptr<Metrics> _metrics;
|
||||
|
||||
// properties map: {name -> [supported, mutable, eval function]}
|
||||
std::map<std::string, std::tuple<bool, ov::PropertyMutability, std::function<ov::Any(const Config&)>>> _properties;
|
||||
std::vector<ov::PropertyName> _supportedProperties;
|
||||
|
||||
static std::atomic<int> _compiledModelLoadCounter;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "async_infer_request.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
// clang-format off
|
||||
AsyncInferRequest::AsyncInferRequest(const std::shared_ptr<SyncInferRequest>& syncInferRequest,
|
||||
const std::shared_ptr<ov::threading::ITaskExecutor>& requestExecutor,
|
||||
const std::shared_ptr<ov::threading::ITaskExecutor>& getResultExecutor,
|
||||
const std::shared_ptr<ov::threading::ITaskExecutor>& callbackExecutor)
|
||||
: ov::IAsyncInferRequest(syncInferRequest, requestExecutor, callbackExecutor),
|
||||
_syncInferRequest(syncInferRequest), _getResultExecutor(getResultExecutor) {
|
||||
m_pipeline = {
|
||||
{requestExecutor, [this] { _syncInferRequest->infer_async(); }},
|
||||
{getResultExecutor, [this] { _syncInferRequest->get_result(); }}
|
||||
};
|
||||
}
|
||||
// clang-format on
|
||||
|
||||
AsyncInferRequest::~AsyncInferRequest() {
|
||||
stop_and_wait();
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,330 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "compiled_model.hpp"
|
||||
|
||||
#include <fstream>
|
||||
#include <string_view>
|
||||
|
||||
#include "async_infer_request.hpp"
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
#include "intel_npu/al/config/compiler.hpp"
|
||||
#include "intel_npu/al/config/config.hpp"
|
||||
#include "intel_npu/al/config/runtime.hpp"
|
||||
#include "intel_npu/al/icompiler.hpp"
|
||||
#include "intel_npu/al/itt.hpp"
|
||||
#include "openvino/pass/constant_folding.hpp"
|
||||
#include "openvino/pass/manager.hpp"
|
||||
#include "openvino/runtime/properties.hpp"
|
||||
#include "openvino/runtime/system_conf.hpp"
|
||||
#include "openvino/runtime/threading/executor_manager.hpp"
|
||||
#include "transformations/utils/utils.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::string_view NO_EXECUTOR_FOR_INFERENCE =
|
||||
"Can't create infer request!\n"
|
||||
"Please make sure that the device is available. Only exports can be made.";
|
||||
|
||||
std::uint32_t hash(const std::vector<uint8_t>& data) {
|
||||
std::uint32_t result = 1171117u;
|
||||
for (const auto& c : data)
|
||||
result = ((result << 7) + result) + static_cast<uint32_t>(c);
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
using intel_npu::envVarStrToBool;
|
||||
|
||||
CompiledModel::CompiledModel(const std::shared_ptr<const ov::Model>& model,
|
||||
const std::shared_ptr<const ov::IPlugin>& plugin,
|
||||
const std::shared_ptr<const NetworkDescription>& networkDescription,
|
||||
const std::shared_ptr<IDevice>& device,
|
||||
const std::optional<ov::SoPtr<ICompiler>>& compiler,
|
||||
const Config& config)
|
||||
: ICompiledModel(model, plugin),
|
||||
_networkPtr(networkDescription),
|
||||
_model(model),
|
||||
_config(config),
|
||||
_logger("CompiledModel", config.get<LOG_LEVEL>()),
|
||||
_device(device),
|
||||
_compiler(compiler) {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::NPUPlugin, "CompiledModel::CompiledModel");
|
||||
|
||||
if (_networkPtr == nullptr) {
|
||||
OPENVINO_THROW("Network is null!");
|
||||
}
|
||||
|
||||
OV_ITT_TASK_CHAIN(COMPILED_MODEL, itt::domains::NPUPlugin, "CompiledModel::CompiledModel", "initialize_properties");
|
||||
initialize_properties();
|
||||
configure_stream_executors();
|
||||
|
||||
OV_ITT_TASK_NEXT(COMPILED_MODEL, "create_executor");
|
||||
const bool configCreateExecutor = _config.get<CREATE_EXECUTOR>();
|
||||
static const auto envVar = std::getenv("IE_NPU_CREATE_EXECUTOR");
|
||||
const bool IE_NPU_CREATE_EXECUTOR =
|
||||
envVar ? envVarStrToBool("IE_NPU_CREATE_EXECUTOR", envVar) : configCreateExecutor;
|
||||
|
||||
if (IE_NPU_CREATE_EXECUTOR) {
|
||||
_logger.info("Creating the executor inside the \"CompiledModel\" constructor");
|
||||
|
||||
// If no device has been defined, the executor shall keep the default value of "nullptr". In this scenario,
|
||||
// only export operations will be allowed
|
||||
if (_device != nullptr) {
|
||||
_executorPtr = _device->createExecutor(_networkPtr, _config);
|
||||
}
|
||||
} else {
|
||||
_logger.info("Executor will not be created inside the \"CompiledModel\" constructor");
|
||||
}
|
||||
|
||||
OV_ITT_TASK_SKIP(COMPILED_MODEL);
|
||||
}
|
||||
|
||||
std::shared_ptr<ov::IAsyncInferRequest> CompiledModel::create_infer_request() const {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::NPUPlugin, "CompiledModel::create_infer_request");
|
||||
|
||||
if (_executorPtr == nullptr && _device != nullptr) {
|
||||
_executorPtr = _device->createExecutor(_networkPtr, _config);
|
||||
}
|
||||
if (_executorPtr == nullptr) {
|
||||
OPENVINO_THROW(NO_EXECUTOR_FOR_INFERENCE);
|
||||
}
|
||||
|
||||
const std::shared_ptr<SyncInferRequest>& syncInferRequest =
|
||||
_device->createInferRequest(shared_from_this(), _executorPtr, _config);
|
||||
syncInferRequest->initialize_states();
|
||||
|
||||
return std::make_shared<AsyncInferRequest>(syncInferRequest,
|
||||
get_task_executor(),
|
||||
_resultExecutor,
|
||||
get_callback_executor());
|
||||
}
|
||||
|
||||
std::shared_ptr<ov::ISyncInferRequest> CompiledModel::create_sync_infer_request() const {
|
||||
OPENVINO_THROW_NOT_IMPLEMENTED(
|
||||
"The synchronous inference request structure implemented by the NPU plugin does not inherit "
|
||||
"the \"ov::ISyncInferRequest\" class");
|
||||
}
|
||||
|
||||
void CompiledModel::export_model(std::ostream& stream) const {
|
||||
const auto& blob = _networkPtr->compiledNetwork;
|
||||
stream.write(reinterpret_cast<const char*>(blob.data()), blob.size());
|
||||
|
||||
std::stringstream str;
|
||||
str << "Blob size: " << blob.size() << ", hash: " << std::hex << hash(blob);
|
||||
_logger.info(str.str().c_str());
|
||||
}
|
||||
|
||||
std::shared_ptr<const ov::Model> CompiledModel::get_runtime_model() const {
|
||||
return _model;
|
||||
}
|
||||
|
||||
void CompiledModel::set_property(const ov::AnyMap& /*properties*/) {
|
||||
OPENVINO_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
ov::Any CompiledModel::get_property(const std::string& name) const {
|
||||
auto configIterator = _properties.find(name);
|
||||
if (configIterator != _properties.cend()) {
|
||||
return std::get<2>(configIterator->second)(_config);
|
||||
}
|
||||
|
||||
OPENVINO_THROW("Unsupported property ", name);
|
||||
}
|
||||
|
||||
const std::shared_ptr<const NetworkDescription>& CompiledModel::get_network_description() const {
|
||||
return _networkPtr;
|
||||
}
|
||||
|
||||
const Config& CompiledModel::get_config() const {
|
||||
return _config;
|
||||
}
|
||||
|
||||
const ov::SoPtr<ICompiler>& CompiledModel::get_compiler() const {
|
||||
if (_compiler.has_value()) {
|
||||
return _compiler.value();
|
||||
}
|
||||
OPENVINO_THROW("PERF_COUNT property is not set");
|
||||
}
|
||||
|
||||
void CompiledModel::configure_stream_executors() {
|
||||
std::shared_ptr<ov::threading::ITaskExecutor> task_executor;
|
||||
if (get_plugin()->get_property(ov::internal::exclusive_async_requests.name(), {}).as<bool>()) {
|
||||
task_executor = ov::threading::executor_manager()->get_executor("NPU");
|
||||
} else if (get_property(ov::hint::enable_cpu_pinning.name()).as<bool>()) {
|
||||
auto executor_config = ov::threading::IStreamsExecutor::Config{
|
||||
"Intel NPU plugin executor",
|
||||
0,
|
||||
0,
|
||||
ov::threading::IStreamsExecutor::ThreadBindingType::CORES,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
ov::threading::IStreamsExecutor::Config::PreferredCoreType::BIG,
|
||||
{{get_plugin()->get_property(ov::num_streams.name(), {}).as<ov::streams::Num>(),
|
||||
ov::MAIN_CORE_PROC,
|
||||
1,
|
||||
0,
|
||||
0}},
|
||||
true};
|
||||
auto post_config = ov::threading::IStreamsExecutor::Config::reserve_cpu_threads(executor_config);
|
||||
task_executor = std::make_shared<ov::threading::CPUStreamsExecutor>(post_config);
|
||||
} else {
|
||||
task_executor = std::make_shared<ov::threading::CPUStreamsExecutor>(
|
||||
ov::threading::IStreamsExecutor::Config{"NPUPlugin executor"});
|
||||
}
|
||||
|
||||
set_task_executor(task_executor);
|
||||
const auto executorId = _networkPtr->metadata.name + "_NPUResultExecutor";
|
||||
_resultExecutor = ov::threading::executor_manager()->get_executor(executorId);
|
||||
}
|
||||
|
||||
void CompiledModel::initialize_properties() {
|
||||
_properties = {
|
||||
// OV Public
|
||||
// =========
|
||||
{ov::supported_properties.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
return _supportedProperties;
|
||||
}}},
|
||||
{ov::device::id.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<DEVICE_ID>();
|
||||
}}},
|
||||
{ov::enable_profiling.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<PERF_COUNT>();
|
||||
}}},
|
||||
{ov::model_name.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
OPENVINO_ASSERT(_networkPtr != nullptr, "Missing network descriptor");
|
||||
return _networkPtr->metadata.name;
|
||||
}}},
|
||||
{ov::optimal_number_of_infer_requests.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config& config) {
|
||||
// value is allowed to be queried prior the network is compiled
|
||||
return static_cast<uint32_t>(getOptimalNumberOfInferRequestsInParallel(config));
|
||||
}}},
|
||||
{ov::execution_devices.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config& config) {
|
||||
// This mainly concerns the device name displayed to the user
|
||||
// e.g. "NPU.3720" or "NPU" if the platform is set to "AUTO_DETECT"
|
||||
if (config.get<PLATFORM>() == ov::intel_npu::Platform::AUTO_DETECT) {
|
||||
return std::string("NPU");
|
||||
}
|
||||
OPENVINO_ASSERT(_device != nullptr, "GetMetric: the device is not initialized");
|
||||
return std::string("NPU.") + _device->getName();
|
||||
}}},
|
||||
{ov::loaded_from_cache.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<LOADED_FROM_CACHE>();
|
||||
}}},
|
||||
// OV Public Hints
|
||||
// =========
|
||||
{ov::hint::performance_mode.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<PERFORMANCE_HINT>();
|
||||
}}},
|
||||
{ov::hint::num_requests.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<PERFORMANCE_HINT_NUM_REQUESTS>();
|
||||
}}},
|
||||
{ov::hint::inference_precision.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<INFERENCE_PRECISION_HINT>();
|
||||
}}},
|
||||
{ov::hint::enable_cpu_pinning.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<ENABLE_CPU_PINNING>();
|
||||
}}},
|
||||
// OV Internals
|
||||
// =========
|
||||
{ov::internal::supported_properties.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
static const std::vector<ov::PropertyName> supportedProperty{
|
||||
ov::PropertyName(ov::internal::caching_properties.name(), ov::PropertyMutability::RO),
|
||||
};
|
||||
return supportedProperty;
|
||||
}}},
|
||||
// NPU Private
|
||||
// =========
|
||||
{ov::hint::model_priority.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<MODEL_PRIORITY>();
|
||||
}}},
|
||||
{ov::intel_npu::tiles.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<TILES>();
|
||||
}}},
|
||||
{ov::intel_npu::profiling_type.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.getString<PROFILING_TYPE>();
|
||||
}}},
|
||||
{ov::intel_npu::platform.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<PLATFORM>();
|
||||
}}},
|
||||
{ov::intel_npu::dynamic_shape_to_static.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.getString<DYNAMIC_SHAPE_TO_STATIC>();
|
||||
}}},
|
||||
{ov::intel_npu::use_elf_compiler_backend.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.getString<USE_ELF_COMPILER_BACKEND>();
|
||||
}}},
|
||||
{ov::intel_npu::create_executor.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<CREATE_EXECUTOR>();
|
||||
}}},
|
||||
};
|
||||
|
||||
for (auto& property : _properties) {
|
||||
if (std::get<0>(property.second)) {
|
||||
_supportedProperties.emplace_back(property.first, std::get<1>(property.second));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,649 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "plugin.hpp"
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include "compiled_model.hpp"
|
||||
#include "device_helpers.hpp"
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
#include "intel_npu/al/config/compiler.hpp"
|
||||
#include "intel_npu/al/config/runtime.hpp"
|
||||
#include "intel_npu/al/itt.hpp"
|
||||
#include "npu_compiler.hpp"
|
||||
#include "npu_metrics.hpp"
|
||||
#include "openvino/op/constant.hpp"
|
||||
#include "openvino/op/parameter.hpp"
|
||||
#include "openvino/runtime/intel_npu/properties.hpp"
|
||||
|
||||
using namespace intel_npu;
|
||||
|
||||
namespace {
|
||||
|
||||
const std::vector<size_t> CONSTANT_NODE_DUMMY_SHAPE{1};
|
||||
|
||||
const char* NPU_PLUGIN_LIB_NAME = "openvino_intel_npu_plugin";
|
||||
|
||||
/**
|
||||
* @brief Creates an "ov::Model" object which contains only the given "parameter" and "result" nodes.
|
||||
* @details Using an "ov::Model" object to create the "CompiledModel" is the preferred way of using the OV API.
|
||||
* This path allows making use of the already written funtions/attributes for handling the I/O infromation.
|
||||
*
|
||||
* Note that a stored compiled model does not hold the original IR model within it. The only related information
|
||||
* which may be extracted is the original model's "parameter"/"result" nodes. Thus, we need to build a dummy model
|
||||
* starting from these fields in order to satisfy the API.
|
||||
* @param parameterDescriptors Describes the input nodes.
|
||||
* @param resultDescriptors Describes the output nodes.
|
||||
* @param inputNames The names of the inputs registered in the order given by the model.
|
||||
* @param outputNames The names of the outputs registered in the order given by the model.
|
||||
*/
|
||||
std::shared_ptr<ov::Model> create_dummy_model(const IONodeDescriptorMap& parameterDescriptors,
|
||||
const IONodeDescriptorMap& resultDescriptors,
|
||||
const std::vector<std::string>& inputNames,
|
||||
const std::vector<std::string>& outputNames) {
|
||||
ov::ParameterVector parameters;
|
||||
ov::NodeVector results;
|
||||
|
||||
for (const std::string& inputName : inputNames) {
|
||||
const IONodeDescriptor& parameterDescriptor = parameterDescriptors.at(inputName);
|
||||
std::shared_ptr<ov::op::v0::Parameter> parameter =
|
||||
std::make_shared<ov::op::v0::Parameter>(parameterDescriptor.precision, parameterDescriptor.transposedShape);
|
||||
parameter->set_friendly_name(parameterDescriptor.currentNodeName);
|
||||
parameter->output(0).get_tensor().set_names(parameterDescriptor.outputTensorNames);
|
||||
parameters.push_back(parameter);
|
||||
}
|
||||
|
||||
// The "result" nodes require a parent node in order to satisfy the legacy API naming conventions as well (in
|
||||
// the 1.0 API, the name of an output is given by the parent of the "result" node). Additionally, a dummy shape for
|
||||
// the "Constant" node was required since the specific constructor does not accept "ov::PartialShape" values (a
|
||||
// constant can't have dynamic shape). The dummy tensor was also brought in order to register the correct,
|
||||
// potentially dynamic, output shape.
|
||||
for (const std::string& outputName : outputNames) {
|
||||
const IONodeDescriptor& resultDescriptor = resultDescriptors.at(outputName);
|
||||
std::shared_ptr<ov::Node> constantDummy =
|
||||
std::make_shared<ov::op::v0::Constant>(resultDescriptor.precision, CONSTANT_NODE_DUMMY_SHAPE);
|
||||
constantDummy->set_friendly_name(resultDescriptor.legacyName);
|
||||
|
||||
const std::shared_ptr<ov::descriptor::Tensor>& tensorDummy =
|
||||
std::make_shared<ov::descriptor::Tensor>(resultDescriptor.precision,
|
||||
resultDescriptor.transposedShape,
|
||||
resultDescriptor.outputTensorNames);
|
||||
|
||||
std::shared_ptr<ov::Node> result = std::make_shared<ov::op::v0::Result>(constantDummy);
|
||||
result->output(0).set_tensor_ptr(tensorDummy);
|
||||
result->set_friendly_name(resultDescriptor.currentNodeName);
|
||||
results.push_back(result);
|
||||
}
|
||||
|
||||
return std::make_shared<ov::Model>(results, parameters);
|
||||
}
|
||||
|
||||
std::map<std::string, std::string> any_copy(const ov::AnyMap& params) {
|
||||
std::map<std::string, std::string> result;
|
||||
for (auto&& value : params) {
|
||||
result.emplace(value.first, value.second.as<std::string>());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
size_t getFileSize(std::istream& stream) {
|
||||
const size_t streamStart = stream.tellg();
|
||||
stream.seekg(0, std::ios_base::end);
|
||||
const size_t streamEnd = stream.tellg();
|
||||
stream.seekg(streamStart, std::ios_base::beg);
|
||||
return streamEnd - streamStart;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
static Config merge_configs(const Config& globalConfig,
|
||||
const std::map<std::string, std::string>& rawConfig,
|
||||
OptionMode mode = OptionMode::Both) {
|
||||
Config localConfig = globalConfig;
|
||||
localConfig.update(rawConfig, mode);
|
||||
return localConfig;
|
||||
}
|
||||
|
||||
static auto get_specified_device_name(const Config config) {
|
||||
if (config.has<DEVICE_ID>()) {
|
||||
return config.get<DEVICE_ID>();
|
||||
}
|
||||
return std::string();
|
||||
}
|
||||
|
||||
static Config add_platform_to_the_config(Config config, const std::string_view platform) {
|
||||
config.update({{ov::intel_npu::platform.name(), std::string(platform)}});
|
||||
return config;
|
||||
}
|
||||
|
||||
Plugin::Plugin()
|
||||
: _options(std::make_shared<OptionsDesc>()),
|
||||
_globalConfig(_options),
|
||||
_logger("NPUPlugin", Logger::global().level()) {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::NPUPlugin, "Plugin::Plugin");
|
||||
set_device_name("NPU");
|
||||
|
||||
registerCommonOptions(*_options);
|
||||
registerCompilerOptions(*_options);
|
||||
registerRunTimeOptions(*_options);
|
||||
|
||||
// parse env_variables to get LOG_LEVEL if needed
|
||||
_globalConfig.parseEnvVars();
|
||||
Logger::global().setLevel(_globalConfig.get<LOG_LEVEL>());
|
||||
|
||||
// TODO: generation of available backends list can be done during execution of CMake scripts
|
||||
std::vector<AvailableBackends> backendRegistry;
|
||||
|
||||
#if defined(OPENVINO_STATIC_LIBRARY)
|
||||
backendRegistry.push_back(AvailableBackends::LEVEL_ZERO);
|
||||
#else
|
||||
# if defined(ENABLE_IMD_BACKEND)
|
||||
if (const auto* envVar = std::getenv("IE_NPU_USE_IMD_BACKEND")) {
|
||||
if (envVarStrToBool("IE_NPU_USE_IMD_BACKEND", envVar)) {
|
||||
backendRegistry.push_back(AvailableBackends::IMD);
|
||||
}
|
||||
}
|
||||
# endif
|
||||
|
||||
# if defined(_WIN32) || defined(_WIN64) || (defined(__linux__) && defined(__x86_64__))
|
||||
backendRegistry.push_back(AvailableBackends::LEVEL_ZERO);
|
||||
# endif
|
||||
#endif
|
||||
|
||||
OV_ITT_TASK_CHAIN(PLUGIN, itt::domains::NPUPlugin, "Plugin::Plugin", "NPUBackends");
|
||||
_backends = std::make_shared<NPUBackends>(backendRegistry, _globalConfig);
|
||||
OV_ITT_TASK_NEXT(PLUGIN, "registerOptions");
|
||||
_backends->registerOptions(*_options);
|
||||
|
||||
OV_ITT_TASK_NEXT(PLUGIN, "Metrics");
|
||||
_metrics = std::make_unique<Metrics>(_backends);
|
||||
|
||||
// parse again env_variables after backend is initialized to get backend proprieties
|
||||
_globalConfig.parseEnvVars();
|
||||
|
||||
// Map from name to function {Config -> ov::Any}
|
||||
// Note that some properties are RW before network is loaded, and become RO after network is loaded
|
||||
_properties = {
|
||||
// OV Public
|
||||
// =========
|
||||
{ov::supported_properties.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
return _supportedProperties;
|
||||
}}},
|
||||
{ov::enable_profiling.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<PERF_COUNT>();
|
||||
}}},
|
||||
{ov::hint::performance_mode.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<PERFORMANCE_HINT>();
|
||||
}}},
|
||||
{ov::hint::num_requests.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<PERFORMANCE_HINT_NUM_REQUESTS>();
|
||||
}}},
|
||||
{ov::hint::inference_precision.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<INFERENCE_PRECISION_HINT>();
|
||||
}}},
|
||||
{ov::hint::enable_cpu_pinning.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<ENABLE_CPU_PINNING>();
|
||||
}}},
|
||||
{ov::log::level.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<LOG_LEVEL>();
|
||||
}}},
|
||||
{ov::cache_dir.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<CACHE_DIR>();
|
||||
}}},
|
||||
{ov::device::id.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<DEVICE_ID>();
|
||||
}}},
|
||||
{ov::compilation_num_threads.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.getString<COMPILATION_NUM_THREADS>();
|
||||
}}},
|
||||
{ov::available_devices.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
return _metrics->GetAvailableDevicesNames();
|
||||
}}},
|
||||
{ov::device::capabilities.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
return _metrics->GetOptimizationCapabilities();
|
||||
}}},
|
||||
{ov::optimal_number_of_infer_requests.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config& config) {
|
||||
return static_cast<uint32_t>(getOptimalNumberOfInferRequestsInParallel(add_platform_to_the_config(
|
||||
config,
|
||||
_backends->getCompilationPlatform(config.get<PLATFORM>(), config.get<DEVICE_ID>()))));
|
||||
}}},
|
||||
{ov::range_for_async_infer_requests.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
return _metrics->GetRangeForAsyncInferRequest();
|
||||
}}},
|
||||
{ov::range_for_streams.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
return _metrics->GetRangeForStreams();
|
||||
}}},
|
||||
{ov::num_streams.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[](const Config& config) {
|
||||
return config.get<NUM_STREAMS>();
|
||||
}}},
|
||||
{ov::device::uuid.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config& config) {
|
||||
const auto specifiedDeviceName = get_specified_device_name(config);
|
||||
auto devUuid = _metrics->GetDeviceUuid(specifiedDeviceName);
|
||||
return decltype(ov::device::uuid)::value_type{devUuid};
|
||||
}}},
|
||||
// Add FULL_DEVICE_NAME and DEVICE_ARCHITECTURE in supported
|
||||
// properties list only in case of non-empty device list (#1424144d)
|
||||
{ov::device::architecture.name(),
|
||||
{!_metrics->GetAvailableDevicesNames().empty(),
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config& config) {
|
||||
const auto specifiedDeviceName = get_specified_device_name(config);
|
||||
return _metrics->GetDeviceArchitecture(specifiedDeviceName);
|
||||
}}},
|
||||
{ov::device::full_name.name(),
|
||||
{!_metrics->GetAvailableDevicesNames().empty(),
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config& config) {
|
||||
const auto specifiedDeviceName = get_specified_device_name(config);
|
||||
return _metrics->GetFullDeviceName(specifiedDeviceName);
|
||||
}}},
|
||||
// OV Internals
|
||||
// =========
|
||||
{ov::internal::caching_properties.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
return _metrics->GetCachingProperties();
|
||||
}}},
|
||||
{ov::internal::exclusive_async_requests.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<EXCLUSIVE_ASYNC_REQUESTS>();
|
||||
}}},
|
||||
{ov::internal::supported_properties.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
return _metrics->GetInternalSupportedProperties();
|
||||
}}},
|
||||
// NPU Public
|
||||
// =========
|
||||
{ov::intel_npu::device_alloc_mem_size.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config& config) {
|
||||
return _metrics->GetDeviceAllocMemSize(get_specified_device_name(config));
|
||||
}}},
|
||||
{ov::intel_npu::device_total_mem_size.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config& config) {
|
||||
return _metrics->GetDeviceTotalMemSize(get_specified_device_name(config));
|
||||
}}},
|
||||
{ov::intel_npu::driver_version.name(),
|
||||
{true,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config& config) {
|
||||
return _metrics->GetDriverVersion(get_specified_device_name(config));
|
||||
}}},
|
||||
// NPU Private
|
||||
// =========
|
||||
{ov::hint::model_priority.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<MODEL_PRIORITY>();
|
||||
}}},
|
||||
{ov::intel_npu::dma_engines.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<DMA_ENGINES>();
|
||||
}}},
|
||||
{ov::intel_npu::tiles.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<TILES>();
|
||||
}}},
|
||||
{ov::intel_npu::dpu_groups.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<DPU_GROUPS>();
|
||||
}}},
|
||||
{ov::intel_npu::stepping.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<STEPPING>();
|
||||
}}},
|
||||
{ov::intel_npu::max_tiles.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<MAX_TILES>();
|
||||
}}},
|
||||
{ov::intel_npu::compilation_mode.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<COMPILATION_MODE>();
|
||||
}}},
|
||||
{ov::intel_npu::compilation_mode_params.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<COMPILATION_MODE_PARAMS>();
|
||||
}}},
|
||||
{ov::intel_npu::compiler_type.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.getString<COMPILER_TYPE>();
|
||||
}}},
|
||||
{ov::intel_npu::platform.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<PLATFORM>();
|
||||
}}},
|
||||
{ov::intel_npu::backend_name.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RO,
|
||||
[&](const Config&) {
|
||||
return _metrics->GetBackendName();
|
||||
}}},
|
||||
{ov::intel_npu::use_elf_compiler_backend.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.getString<USE_ELF_COMPILER_BACKEND>();
|
||||
}}},
|
||||
{ov::intel_npu::dynamic_shape_to_static.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<DYNAMIC_SHAPE_TO_STATIC>();
|
||||
}}},
|
||||
{ov::intel_npu::profiling_type.name(),
|
||||
{false,
|
||||
ov::PropertyMutability::RW,
|
||||
[](const Config& config) {
|
||||
return config.get<PROFILING_TYPE>();
|
||||
}}},
|
||||
};
|
||||
|
||||
for (auto& property : _properties) {
|
||||
if (std::get<0>(property.second)) {
|
||||
_supportedProperties.emplace_back(ov::PropertyName(property.first, std::get<1>(property.second)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Plugin::set_property(const ov::AnyMap& properties) {
|
||||
const std::map<std::string, std::string> config = any_copy(properties);
|
||||
for (const auto& configEntry : config) {
|
||||
if (_properties.find(configEntry.first) == _properties.end()) {
|
||||
OPENVINO_THROW("Unsupported configuration key: ", configEntry.first);
|
||||
} else {
|
||||
if (std::get<1>(_properties[configEntry.first]) == ov::PropertyMutability::RO) {
|
||||
OPENVINO_THROW("READ-ONLY configuration key: ", configEntry.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_globalConfig.update(config);
|
||||
Logger::global().setLevel(_globalConfig.get<LOG_LEVEL>());
|
||||
if (_backends != nullptr) {
|
||||
_backends->setup(_globalConfig);
|
||||
}
|
||||
|
||||
for (const auto& entry : config) {
|
||||
_config[entry.first] = entry.second;
|
||||
}
|
||||
}
|
||||
|
||||
ov::Any Plugin::get_property(const std::string& name, const ov::AnyMap& arguments) const {
|
||||
const std::map<std::string, std::string>& amends = any_copy(arguments);
|
||||
const Config amendedConfig = merge_configs(_globalConfig, amends);
|
||||
|
||||
auto&& configIterator = _properties.find(name);
|
||||
if (configIterator != _properties.cend()) {
|
||||
return std::get<2>(configIterator->second)(amendedConfig);
|
||||
}
|
||||
|
||||
OPENVINO_THROW("Unsupported configuration key: ", name);
|
||||
}
|
||||
|
||||
std::shared_ptr<ov::ICompiledModel> Plugin::compile_model(const std::shared_ptr<const ov::Model>& model,
|
||||
const ov::AnyMap& properties) const {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::NPUPlugin, "Plugin::compile_model");
|
||||
OV_ITT_TASK_CHAIN(PLUGIN_COMPILE_MODEL, itt::domains::NPUPlugin, "Plugin::compile_model", "merge_configs");
|
||||
auto localConfig = merge_configs(_globalConfig, any_copy(properties));
|
||||
|
||||
const auto set_cache_dir = localConfig.get<CACHE_DIR>();
|
||||
if (!set_cache_dir.empty()) {
|
||||
const auto compilerType = localConfig.get<COMPILER_TYPE>();
|
||||
if (compilerType == ov::intel_npu::CompilerType::MLIR) {
|
||||
_logger.error("Option 'CACHE_DIR' is not supported with MLIR compiler type");
|
||||
}
|
||||
}
|
||||
|
||||
const auto platform = _backends->getCompilationPlatform(localConfig.get<PLATFORM>(), localConfig.get<DEVICE_ID>());
|
||||
auto device = _backends->getDevice(localConfig.get<DEVICE_ID>());
|
||||
localConfig.update({{ov::intel_npu::platform.name(), platform}});
|
||||
|
||||
// Update stepping w/ information from driver, unless provided by user or we are off-device
|
||||
// Ignore, if compilation was requested for platform, different from current
|
||||
if (!localConfig.has<STEPPING>() && device != nullptr && device->getName() == platform) {
|
||||
try {
|
||||
localConfig.update({{ov::intel_npu::stepping.name(), std::to_string(device->getSubDevId())}});
|
||||
} catch (...) {
|
||||
_logger.warning("Stepping information not implemented by selected backend. Skipping. Please provide "
|
||||
"NPU_STEPPING if required.");
|
||||
}
|
||||
}
|
||||
// Update max_tiles w/ information from driver, unless provided by user or we are off-device
|
||||
// Ignore, if compilation was requested for platform, different from current
|
||||
if (!localConfig.has<MAX_TILES>() && device != nullptr && device->getName() == platform) {
|
||||
try {
|
||||
localConfig.update({{ov::intel_npu::max_tiles.name(), std::to_string(device->getMaxNumSlices())}});
|
||||
} catch (...) {
|
||||
_logger.warning("Max tiles information not implemented by selected backend. Skipping. Please provide "
|
||||
"NPU_MAX_TILES if required.");
|
||||
}
|
||||
}
|
||||
|
||||
OV_ITT_TASK_NEXT(PLUGIN_COMPILE_MODEL, "compile");
|
||||
|
||||
std::shared_ptr<const NetworkDescription> networkDescription;
|
||||
std::shared_ptr<ov::ICompiledModel> compiledModel;
|
||||
|
||||
ov::SoPtr<ICompiler> compiler;
|
||||
try {
|
||||
compiler = getCompiler(localConfig);
|
||||
networkDescription = std::make_shared<const NetworkDescription>(compiler->compile(model, localConfig));
|
||||
} catch (const std::exception& ex) {
|
||||
OPENVINO_THROW(ex.what());
|
||||
} catch (...) {
|
||||
_logger.error("Unexpected exception");
|
||||
OPENVINO_THROW("NPU ExecutableNetwork got unexpected exception from compiler");
|
||||
}
|
||||
|
||||
try {
|
||||
bool profiling = localConfig.get<PERF_COUNT>();
|
||||
|
||||
compiledModel = std::make_shared<CompiledModel>(model,
|
||||
shared_from_this(),
|
||||
networkDescription,
|
||||
device,
|
||||
profiling ? std::optional(compiler) : std::nullopt,
|
||||
localConfig);
|
||||
} catch (const std::exception& ex) {
|
||||
OPENVINO_THROW(ex.what());
|
||||
} catch (...) {
|
||||
OPENVINO_THROW("Unexpected exception thrown upon attempting to create the \"CompiledModel\" object");
|
||||
}
|
||||
|
||||
++_compiledModelLoadCounter;
|
||||
OV_ITT_TASK_SKIP(PLUGIN_COMPILE_MODEL);
|
||||
|
||||
return compiledModel;
|
||||
}
|
||||
|
||||
std::shared_ptr<ov::ICompiledModel> Plugin::compile_model(const std::shared_ptr<const ov::Model>& /*model*/,
|
||||
const ov::AnyMap& /*properties*/,
|
||||
const ov::SoPtr<ov::IRemoteContext>& /*context*/) const {
|
||||
OPENVINO_THROW_NOT_IMPLEMENTED("The remote context feature is not supported by the NPU plugin");
|
||||
}
|
||||
|
||||
ov::SoPtr<ov::IRemoteContext> Plugin::create_context(const ov::AnyMap& /*remote_properties*/) const {
|
||||
OPENVINO_THROW_NOT_IMPLEMENTED("The remote context feature is not supported by the NPU plugin");
|
||||
}
|
||||
|
||||
ov::SoPtr<ov::IRemoteContext> Plugin::get_default_context(const ov::AnyMap& /*remote_properties*/) const {
|
||||
OPENVINO_THROW_NOT_IMPLEMENTED("The remote context feature is not supported by the NPU plugin");
|
||||
}
|
||||
|
||||
std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& stream, const ov::AnyMap& properties) const {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::NPUPlugin, "Plugin::import_model");
|
||||
|
||||
OV_ITT_TASK_CHAIN(PLUGIN_IMPORT_MODEL, itt::domains::NPUPlugin, "Plugin::import_model", "merge_configs");
|
||||
auto localConfig = merge_configs(_globalConfig, any_copy(properties), OptionMode::RunTime);
|
||||
const auto platform = _backends->getCompilationPlatform(localConfig.get<PLATFORM>(), localConfig.get<DEVICE_ID>());
|
||||
localConfig.update({{ov::intel_npu::platform.name(), platform}});
|
||||
auto device = _backends->getDevice(localConfig.get<DEVICE_ID>());
|
||||
|
||||
Logger logger("NPUPlugin", localConfig.get<LOG_LEVEL>());
|
||||
|
||||
const auto loadedFromCache = localConfig.get<LOADED_FROM_CACHE>();
|
||||
if (!loadedFromCache) {
|
||||
logger.warning("The usage of a compiled model can lead to undefined behavior. Please use OpenVINO IR instead!");
|
||||
}
|
||||
|
||||
OV_ITT_TASK_NEXT(PLUGIN_IMPORT_MODEL, "parse");
|
||||
|
||||
std::shared_ptr<ov::ICompiledModel> compiledModel;
|
||||
|
||||
try {
|
||||
auto compiler = getCompiler(localConfig);
|
||||
|
||||
auto graphSize = getFileSize(stream);
|
||||
if (graphSize == 0) {
|
||||
OPENVINO_THROW("Blob is empty");
|
||||
}
|
||||
std::vector<uint8_t> blob(graphSize);
|
||||
stream.read(reinterpret_cast<char*>(blob.data()), graphSize);
|
||||
|
||||
auto meta = compiler->parse(blob, localConfig);
|
||||
meta.name = "net" + std::to_string(_compiledModelLoadCounter++);
|
||||
|
||||
const std::shared_ptr<ov::Model> modelDummy =
|
||||
create_dummy_model(meta.parameters, meta.results, meta.inputNames, meta.outputNames);
|
||||
|
||||
bool profiling = localConfig.get<PERF_COUNT>();
|
||||
|
||||
auto networkDescription = std::make_shared<const NetworkDescription>(std::move(blob), std::move(meta));
|
||||
|
||||
compiledModel = std::make_shared<CompiledModel>(modelDummy,
|
||||
shared_from_this(),
|
||||
networkDescription,
|
||||
device,
|
||||
profiling ? std::optional(compiler) : std::nullopt,
|
||||
localConfig);
|
||||
} catch (const std::exception& ex) {
|
||||
OPENVINO_THROW("Can't import network: ", ex.what());
|
||||
} catch (...) {
|
||||
OPENVINO_THROW("NPU import_model got unexpected exception from CompiledModel");
|
||||
}
|
||||
|
||||
OV_ITT_TASK_SKIP(PLUGIN_IMPORT_MODEL);
|
||||
|
||||
return compiledModel;
|
||||
}
|
||||
|
||||
std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& /*stream*/,
|
||||
const ov::SoPtr<ov::IRemoteContext>& /*context*/,
|
||||
const ov::AnyMap& /*properties*/) const {
|
||||
OPENVINO_THROW_NOT_IMPLEMENTED("The remote context feature is not supported by the NPU plugin");
|
||||
}
|
||||
|
||||
ov::SupportedOpsMap Plugin::query_model(const std::shared_ptr<const ov::Model>& model,
|
||||
const ov::AnyMap& properties) const {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::NPUPlugin, "Plugin::query_model");
|
||||
|
||||
auto localConfig = merge_configs(_globalConfig, any_copy(properties), OptionMode::CompileTime);
|
||||
const auto platform = _backends->getCompilationPlatform(localConfig.get<PLATFORM>(), localConfig.get<DEVICE_ID>());
|
||||
localConfig.update({{ov::intel_npu::platform.name(), platform}});
|
||||
|
||||
auto compiler = getCompiler(localConfig);
|
||||
ov::SupportedOpsMap supportedOpsMap;
|
||||
try {
|
||||
supportedOpsMap = compiler->query(model, localConfig);
|
||||
} catch (const std::runtime_error& e) {
|
||||
OPENVINO_THROW(e.what());
|
||||
} catch (...) {
|
||||
OPENVINO_THROW("NPU query_model got unexpected error from compiler");
|
||||
}
|
||||
|
||||
return supportedOpsMap;
|
||||
}
|
||||
|
||||
ov::SoPtr<ICompiler> Plugin::getCompiler(const Config& config) const {
|
||||
auto compilerType = config.get<COMPILER_TYPE>();
|
||||
return createCompiler(compilerType, _logger);
|
||||
}
|
||||
|
||||
std::atomic<int> Plugin::_compiledModelLoadCounter{1};
|
||||
|
||||
static const ov::Version version = {CI_BUILD_NUMBER, NPU_PLUGIN_LIB_NAME};
|
||||
OV_DEFINE_PLUGIN_CREATE_FUNCTION(Plugin, version)
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
|
||||
#include "device_helpers.hpp"
|
||||
#include "intel_npu/al/config/common.hpp"
|
||||
#include "npu_backends.hpp"
|
||||
|
||||
#if defined(ENABLE_ZEROAPI_BACKEND)
|
||||
# include "zero_backend.hpp"
|
||||
#endif
|
||||
|
||||
#if !defined(OPENVINO_STATIC_LIBRARY) && defined(ENABLE_IMD_BACKEND)
|
||||
# include "openvino/util/file_util.hpp"
|
||||
# include "openvino/util/shared_object.hpp"
|
||||
#endif
|
||||
|
||||
using namespace intel_npu;
|
||||
|
||||
namespace {
|
||||
|
||||
std::string backendToString(const AvailableBackends& backend) {
|
||||
switch (backend) {
|
||||
case AvailableBackends::LEVEL_ZERO:
|
||||
return "npu_level_zero_backend";
|
||||
case AvailableBackends::IMD:
|
||||
return "npu_imd_backend";
|
||||
default:
|
||||
return "unsupported backend";
|
||||
}
|
||||
}
|
||||
|
||||
#if !defined(OPENVINO_STATIC_LIBRARY) && defined(ENABLE_IMD_BACKEND)
|
||||
std::shared_ptr<void> loadBackendLibrary(const std::string& libpath) {
|
||||
# if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
|
||||
return ov::util::load_shared_object(ov::util::string_to_wstring(libpath).c_str());
|
||||
# else
|
||||
return ov::util::load_shared_object(libpath.c_str());
|
||||
# endif
|
||||
}
|
||||
|
||||
std::shared_ptr<IEngineBackend> getBackend(std::shared_ptr<void> so, const Config& config) {
|
||||
static constexpr auto CreateFuncName = "CreateNPUEngineBackend";
|
||||
auto symbol = ov::util::get_symbol(so, CreateFuncName);
|
||||
|
||||
using CreateFuncT = void (*)(std::shared_ptr<IEngineBackend>&, const Config&);
|
||||
const auto createFunc = reinterpret_cast<CreateFuncT>(symbol);
|
||||
|
||||
std::shared_ptr<IEngineBackend> backendPtr;
|
||||
createFunc(backendPtr, config);
|
||||
return backendPtr;
|
||||
}
|
||||
|
||||
ov::SoPtr<IEngineBackend> loadBackend(const std::string& libpath, const Config& config) {
|
||||
auto backendSO = loadBackendLibrary(libpath);
|
||||
auto backend = getBackend(backendSO, config);
|
||||
|
||||
return ov::SoPtr<IEngineBackend>(backend, backendSO);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
// TODO Config will be useless here, since only default values will be used
|
||||
NPUBackends::NPUBackends(const std::vector<AvailableBackends>& backendRegistry, [[maybe_unused]] const Config& config)
|
||||
: _logger("NPUBackends", Logger::global().level()) {
|
||||
std::vector<ov::SoPtr<IEngineBackend>> registeredBackends;
|
||||
[[maybe_unused]] const auto registerBackend = [&](ov::SoPtr<IEngineBackend> backend, const std::string& name) {
|
||||
const auto backendDevices = backend->getDeviceNames();
|
||||
if (!backendDevices.empty()) {
|
||||
std::stringstream deviceNames;
|
||||
for (const auto& device : backendDevices) {
|
||||
deviceNames << device << " ";
|
||||
}
|
||||
_logger.debug("Register '%s' with devices '%s'", name.c_str(), deviceNames.str().c_str());
|
||||
registeredBackends.emplace_back(backend);
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto& name : backendRegistry) {
|
||||
std::string backendName = backendToString(name);
|
||||
_logger.debug("Try '%s' backend", backendName.c_str());
|
||||
|
||||
try {
|
||||
#if !defined(OPENVINO_STATIC_LIBRARY) && defined(ENABLE_IMD_BACKEND)
|
||||
if (name == AvailableBackends::IMD) {
|
||||
const auto path =
|
||||
ov::util::make_plugin_library_name(ov::util::get_ov_lib_path(), backendName + OV_BUILD_POSTFIX);
|
||||
const auto exists = std::ifstream(path).good();
|
||||
if (!exists) {
|
||||
_logger.debug("Backend '%s' at '%s' doesn't exist", backendName.c_str(), path.c_str());
|
||||
continue;
|
||||
}
|
||||
const auto backend = loadBackend(path, config);
|
||||
registerBackend(backend, backendName);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(ENABLE_ZEROAPI_BACKEND)
|
||||
if (name == AvailableBackends::LEVEL_ZERO) {
|
||||
const auto backend = ov::SoPtr<IEngineBackend>(std::make_shared<ZeroEngineBackend>(config));
|
||||
registerBackend(backend, backendName);
|
||||
}
|
||||
#endif
|
||||
} catch (const std::exception& ex) {
|
||||
_logger.error("Got an error during backend '%s' loading : %s", backendName.c_str(), ex.what());
|
||||
} catch (...) {
|
||||
_logger.error("Got an unknown error during backend '%s' loading", backendName.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
if (registeredBackends.empty()) {
|
||||
registeredBackends.emplace_back(nullptr);
|
||||
}
|
||||
|
||||
// TODO: implementation of getDevice methods needs to be updated to go over all
|
||||
// registered backends to search a device.
|
||||
// A single backend is chosen for now to keep existing behavior
|
||||
_backend = *registeredBackends.begin();
|
||||
|
||||
if (_backend != nullptr) {
|
||||
_logger.info("Use '%s' backend for inference", _backend->getName().c_str());
|
||||
} else {
|
||||
_logger.error("Cannot find backend for inference. Make sure the device is available.");
|
||||
}
|
||||
}
|
||||
|
||||
std::string NPUBackends::getBackendName() const {
|
||||
if (_backend != nullptr) {
|
||||
return _backend->getName();
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
std::shared_ptr<IDevice> NPUBackends::getDevice(const std::string& specificName) const {
|
||||
_logger.debug("Searching for device %s to use started...", specificName.c_str());
|
||||
// TODO iterate over all available backends
|
||||
std::shared_ptr<IDevice> deviceToUse;
|
||||
|
||||
if (_backend != nullptr) {
|
||||
if (specificName.empty()) {
|
||||
deviceToUse = _backend->getDevice();
|
||||
} else {
|
||||
deviceToUse = _backend->getDevice(specificName);
|
||||
}
|
||||
}
|
||||
|
||||
if (deviceToUse == nullptr) {
|
||||
_logger.warning("Device not found!");
|
||||
} else {
|
||||
_logger.debug("Device found: %s", deviceToUse->getName().c_str());
|
||||
}
|
||||
return deviceToUse;
|
||||
}
|
||||
|
||||
std::shared_ptr<IDevice> NPUBackends::getDevice(const ov::AnyMap& paramMap) const {
|
||||
return _backend->getDevice(paramMap);
|
||||
}
|
||||
|
||||
std::vector<std::string> NPUBackends::getAvailableDevicesNames() const {
|
||||
return _backend == nullptr ? std::vector<std::string>() : _backend->getDeviceNames();
|
||||
}
|
||||
|
||||
void NPUBackends::registerOptions(OptionsDesc& options) const {
|
||||
if (_backend != nullptr) {
|
||||
_backend->registerOptions(options);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO config should be also specified to backends, to allow use logging in devices and all levels below
|
||||
void NPUBackends::setup(const Config& config) {
|
||||
_logger.setLevel(config.get<LOG_LEVEL>());
|
||||
}
|
||||
|
||||
std::string NPUBackends::getCompilationPlatform(const std::string_view platform, const std::string& deviceId) const {
|
||||
// Platform parameter has a higher priority than deviceID
|
||||
if (platform != ov::intel_npu::Platform::AUTO_DETECT) {
|
||||
return std::string(platform);
|
||||
}
|
||||
|
||||
// Get compilation platform from deviceID
|
||||
if (!deviceId.empty()) {
|
||||
return utils::getPlatformByDeviceName(deviceId);
|
||||
}
|
||||
|
||||
// Automatic detection of compilation platform
|
||||
const auto devNames = getAvailableDevicesNames();
|
||||
if (devNames.empty()) {
|
||||
OPENVINO_THROW("No NPU devices were found.");
|
||||
}
|
||||
|
||||
return utils::getPlatformByDeviceName(devNames.at(0));
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "intel_npu/al/config/compiler.hpp"
|
||||
#include "intel_npu/al/itt.hpp"
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
#include "npu_compiler.hpp"
|
||||
#include "npu_private_properties.hpp" // AL
|
||||
|
||||
#ifdef ENABLE_DRIVER_COMPILER_ADAPTER
|
||||
# include "npu_driver_compiler_adapter.hpp"
|
||||
#endif
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "openvino/util/file_util.hpp"
|
||||
#include "openvino/util/shared_object.hpp"
|
||||
|
||||
using namespace intel_npu;
|
||||
|
||||
namespace {
|
||||
|
||||
std::shared_ptr<void> loadLibrary(const std::string& libpath) {
|
||||
#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
|
||||
return ov::util::load_shared_object(ov::util::string_to_wstring(libpath).c_str());
|
||||
#else
|
||||
return ov::util::load_shared_object(libpath.c_str());
|
||||
#endif
|
||||
}
|
||||
|
||||
std::shared_ptr<ICompiler> getCompiler(std::shared_ptr<void> so) {
|
||||
static constexpr auto CreateFuncName = "CreateNPUCompiler";
|
||||
auto symbol = ov::util::get_symbol(so, CreateFuncName);
|
||||
|
||||
using CreateFuncT = void (*)(std::shared_ptr<ICompiler>&);
|
||||
const auto createFunc = reinterpret_cast<CreateFuncT>(symbol);
|
||||
|
||||
std::shared_ptr<ICompiler> compilerPtr;
|
||||
createFunc(compilerPtr);
|
||||
return compilerPtr;
|
||||
}
|
||||
|
||||
ov::SoPtr<ICompiler> loadCompiler(const std::string& libpath) {
|
||||
auto compilerSO = loadLibrary(libpath);
|
||||
auto compiler = getCompiler(compilerSO);
|
||||
|
||||
return ov::SoPtr<ICompiler>(compiler, compilerSO);
|
||||
}
|
||||
|
||||
ov::SoPtr<ICompiler> createCompilerAdapter(const Logger& log) {
|
||||
log.info("Driver compiler will be used.");
|
||||
#ifdef ENABLE_DRIVER_COMPILER_ADAPTER
|
||||
const auto compilerInterface = std::make_shared<driverCompilerAdapter::LevelZeroCompilerAdapter>();
|
||||
return ov::SoPtr<ICompiler>(compilerInterface);
|
||||
#else
|
||||
OPENVINO_THROW("NPU Compiler Adapter is not enabled");
|
||||
#endif
|
||||
}
|
||||
|
||||
ov::SoPtr<ICompiler> createNPUCompiler(const Logger& log) {
|
||||
log.info("MLIR compiler will be used.");
|
||||
std::string baseName = "npu_mlir_compiler";
|
||||
auto libPath = ov::util::make_plugin_library_name(ov::util::get_ov_lib_path(), baseName + OV_BUILD_POSTFIX);
|
||||
return loadCompiler(libPath);
|
||||
}
|
||||
|
||||
ov::SoPtr<ICompiler> createCompilerImpl(ov::intel_npu::CompilerType compilerType, const Logger& log) {
|
||||
switch (compilerType) {
|
||||
case ov::intel_npu::CompilerType::MLIR:
|
||||
return createNPUCompiler(log);
|
||||
case ov::intel_npu::CompilerType::DRIVER:
|
||||
return createCompilerAdapter(log);
|
||||
default:
|
||||
OPENVINO_THROW("Invalid NPU_COMPILER_TYPE");
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
ov::SoPtr<ICompiler> intel_npu::createCompiler(ov::intel_npu::CompilerType compilerType, const Logger& log) {
|
||||
OV_ITT_SCOPED_TASK(itt::domains::NPUPlugin, "intel_npu::createCompiler");
|
||||
auto logger = log.clone("createCompiler");
|
||||
try {
|
||||
return createCompilerImpl(compilerType, logger);
|
||||
} catch (const std::exception& ex) {
|
||||
OPENVINO_THROW("Got an error during compiler creation: ", ex.what());
|
||||
} catch (...) {
|
||||
OPENVINO_THROW("Got an unknown error during compiler creation");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
// Plugin
|
||||
#include "device_helpers.hpp"
|
||||
#include "npu_metrics.hpp"
|
||||
#include "npu_private_properties.hpp"
|
||||
#include "openvino/runtime/intel_npu/properties.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
Metrics::Metrics(const std::shared_ptr<const NPUBackends>& backends) : _backends(backends) {
|
||||
_supportedMetrics = {ov::supported_properties.name(),
|
||||
ov::available_devices.name(),
|
||||
ov::device::full_name.name(),
|
||||
ov::device::capabilities.name(),
|
||||
ov::range_for_async_infer_requests.name(),
|
||||
ov::range_for_streams.name(),
|
||||
ov::device::capability::EXPORT_IMPORT,
|
||||
ov::device::architecture.name(),
|
||||
ov::internal::caching_properties.name(),
|
||||
ov::internal::supported_properties.name(),
|
||||
ov::cache_dir.name(),
|
||||
ov::intel_npu::device_alloc_mem_size.name(),
|
||||
ov::intel_npu::device_total_mem_size.name(),
|
||||
ov::intel_npu::driver_version.name()};
|
||||
|
||||
_supportedConfigKeys = {ov::log::level.name(),
|
||||
ov::enable_profiling.name(),
|
||||
ov::device::id.name(),
|
||||
ov::hint::performance_mode.name(),
|
||||
ov::num_streams.name(),
|
||||
ov::hint::num_requests.name(),
|
||||
ov::intel_npu::compilation_mode_params.name(),
|
||||
ov::intel_npu::dynamic_shape_to_static.name()};
|
||||
}
|
||||
|
||||
std::vector<std::string> Metrics::GetAvailableDevicesNames() const {
|
||||
return _backends == nullptr ? std::vector<std::string>() : _backends->getAvailableDevicesNames();
|
||||
}
|
||||
|
||||
// TODO each backend may support different metrics
|
||||
const std::vector<std::string>& Metrics::SupportedMetrics() const {
|
||||
return _supportedMetrics;
|
||||
}
|
||||
|
||||
std::string Metrics::GetFullDeviceName(const std::string& specifiedDeviceName) const {
|
||||
const auto devName = getDeviceName(specifiedDeviceName);
|
||||
auto device = _backends->getDevice(devName);
|
||||
if (device) {
|
||||
return device->getFullDeviceName();
|
||||
}
|
||||
OPENVINO_THROW("No device with name '", specifiedDeviceName, "' is available");
|
||||
}
|
||||
|
||||
// TODO each backend may support different configs
|
||||
const std::vector<std::string>& Metrics::GetSupportedConfigKeys() const {
|
||||
return _supportedConfigKeys;
|
||||
}
|
||||
|
||||
// TODO each backend may support different optimization capabilities
|
||||
const std::vector<std::string>& Metrics::GetOptimizationCapabilities() const {
|
||||
return _optimizationCapabilities;
|
||||
}
|
||||
|
||||
const std::tuple<uint32_t, uint32_t, uint32_t>& Metrics::GetRangeForAsyncInferRequest() const {
|
||||
return _rangeForAsyncInferRequests;
|
||||
}
|
||||
|
||||
const std::tuple<uint32_t, uint32_t>& Metrics::GetRangeForStreams() const {
|
||||
return _rangeForStreams;
|
||||
}
|
||||
|
||||
std::string Metrics::GetDeviceArchitecture(const std::string& specifiedDeviceName) const {
|
||||
const auto devName = getDeviceName(specifiedDeviceName);
|
||||
return utils::getPlatformByDeviceName(devName);
|
||||
}
|
||||
|
||||
IDevice::Uuid Metrics::GetDeviceUuid(const std::string& specifiedDeviceName) const {
|
||||
const auto devName = getDeviceName(specifiedDeviceName);
|
||||
auto device = _backends->getDevice(devName);
|
||||
if (device) {
|
||||
return device->getUuid();
|
||||
}
|
||||
return IDevice::Uuid{};
|
||||
}
|
||||
|
||||
std::vector<ov::PropertyName> Metrics::GetCachingProperties() const {
|
||||
return _cachingProperties;
|
||||
}
|
||||
|
||||
std::vector<ov::PropertyName> Metrics::GetInternalSupportedProperties() const {
|
||||
return _internalSupportedProperties;
|
||||
}
|
||||
|
||||
std::string Metrics::GetBackendName() const {
|
||||
if (_backends == nullptr) {
|
||||
OPENVINO_THROW("No available backends");
|
||||
}
|
||||
|
||||
return _backends->getBackendName();
|
||||
}
|
||||
|
||||
uint64_t Metrics::GetDeviceAllocMemSize(const std::string& specifiedDeviceName) const {
|
||||
const auto devName = getDeviceName(specifiedDeviceName);
|
||||
auto device = _backends->getDevice(devName);
|
||||
if (device) {
|
||||
return device->getAllocMemSize();
|
||||
}
|
||||
OPENVINO_THROW("No device with name '", specifiedDeviceName, "' is available");
|
||||
}
|
||||
|
||||
uint64_t Metrics::GetDeviceTotalMemSize(const std::string& specifiedDeviceName) const {
|
||||
const auto devName = getDeviceName(specifiedDeviceName);
|
||||
auto device = _backends->getDevice(devName);
|
||||
if (device) {
|
||||
return device->getTotalMemSize();
|
||||
}
|
||||
OPENVINO_THROW("No device with name '", specifiedDeviceName, "' is available");
|
||||
}
|
||||
|
||||
uint32_t Metrics::GetDriverVersion(const std::string& specifiedDeviceName) const {
|
||||
const auto devName = getDeviceName(specifiedDeviceName);
|
||||
auto device = _backends->getDevice(devName);
|
||||
if (device) {
|
||||
return device->getDriverVersion();
|
||||
}
|
||||
OPENVINO_THROW("No device with name '", specifiedDeviceName, "' is available");
|
||||
}
|
||||
|
||||
std::string Metrics::getDeviceName(const std::string& specifiedDeviceName) const {
|
||||
std::vector<std::string> devNames;
|
||||
if (_backends == nullptr || (devNames = _backends->getAvailableDevicesNames()).empty()) {
|
||||
OPENVINO_THROW("No available devices");
|
||||
}
|
||||
|
||||
// In case of single device and empty input from user we should use the first element from the device list
|
||||
if (specifiedDeviceName.empty()) {
|
||||
if (devNames.size() == 1) {
|
||||
return devNames[0];
|
||||
} else {
|
||||
OPENVINO_THROW("The device name was not specified. Please specify device name by providing DEVICE_ID");
|
||||
}
|
||||
}
|
||||
|
||||
return specifiedDeviceName;
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#
|
||||
# Copyright (C) 2024 Intel Corporation.
|
||||
# SPDX-License-Identifier: Apache 2.0
|
||||
#
|
||||
|
||||
set(NPU_UTILS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
add_subdirectory(src)
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
//
|
||||
// Class for pretty-logging.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
#include "openvino/runtime/properties.hpp"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
//
|
||||
// Logger
|
||||
//
|
||||
|
||||
std::string printFormattedCStr(const char* fmt, ...)
|
||||
#if defined(__clang__)
|
||||
;
|
||||
#elif defined(__GNUC__) || defined(__GNUG__)
|
||||
__attribute__((format(printf, 1, 2)));
|
||||
#else
|
||||
;
|
||||
#endif
|
||||
|
||||
class Logger {
|
||||
public:
|
||||
static Logger& global();
|
||||
|
||||
Logger(std::string_view name, ov::log::Level lvl = ov::log::Level::NO);
|
||||
Logger(const Logger& log) = default;
|
||||
|
||||
Logger clone(std::string_view name) const;
|
||||
|
||||
auto name() const {
|
||||
return _name;
|
||||
}
|
||||
|
||||
void setName(std::string_view name) {
|
||||
_name = name;
|
||||
}
|
||||
|
||||
auto level() const {
|
||||
return _logLevel;
|
||||
}
|
||||
|
||||
Logger& setLevel(ov::log::Level lvl) {
|
||||
_logLevel = lvl;
|
||||
return *this;
|
||||
}
|
||||
|
||||
bool isActive(ov::log::Level msgLevel) const;
|
||||
|
||||
static std::ostream& getBaseStream();
|
||||
static std::ostream& getLevelStream(ov::log::Level msgLevel);
|
||||
|
||||
template <typename... Args>
|
||||
void error(const char* format, Args&&... args) const {
|
||||
addEntryPacked(ov::log::Level::ERR, format, std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
void warning(const char* format, Args&&... args) const {
|
||||
addEntryPacked(ov::log::Level::WARNING, format, std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
void info(const char* format, Args&&... args) const {
|
||||
addEntryPacked(ov::log::Level::INFO, format, std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
void debug(const char* format, Args&&... args) const {
|
||||
addEntryPacked(ov::log::Level::DEBUG, format, std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
void trace(const char* format, Args&&... args) const {
|
||||
addEntryPacked(ov::log::Level::TRACE, format, std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
private:
|
||||
void addEntryPackedActive(ov::log::Level msgLevel, const std::string_view msg) const;
|
||||
|
||||
template <typename... Args>
|
||||
void addEntryPacked(ov::log::Level msgLevel, const char* format, Args&&... args) const {
|
||||
if (!isActive(msgLevel)) {
|
||||
return;
|
||||
}
|
||||
addEntryPackedActive(msgLevel, printFormattedCStr(format, std::forward<Args>(args)...));
|
||||
}
|
||||
|
||||
void addEntryPacked(ov::log::Level msgLevel, const char* msg) const {
|
||||
if (!isActive(msgLevel)) {
|
||||
return;
|
||||
}
|
||||
addEntryPackedActive(msgLevel, msg);
|
||||
}
|
||||
|
||||
private:
|
||||
std::string _name;
|
||||
ov::log::Level _logLevel = ov::log::Level::NO;
|
||||
};
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ze_api.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
const std::string ze_result_to_string(const ze_result_t result);
|
||||
|
||||
const std::string ze_result_to_description(const ze_result_t result);
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
add_subdirectory(logger)
|
||||
add_subdirectory(zero)
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set(TARGET_NAME openvino_npu_logger_utils)
|
||||
|
||||
file(GLOB_RECURSE SOURCES *.cpp)
|
||||
|
||||
source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES})
|
||||
|
||||
add_library(${TARGET_NAME} STATIC ${SOURCES})
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
|
||||
|
||||
add_library(openvino::npu_logger_utils ALIAS ${TARGET_NAME})
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES EXPORT_NAME npu_logger_utils)
|
||||
|
||||
target_include_directories(${TARGET_NAME}
|
||||
PUBLIC
|
||||
$<BUILD_INTERFACE:${NPU_UTILS_SOURCE_DIR}/include>
|
||||
)
|
||||
|
||||
target_link_libraries(${TARGET_NAME}
|
||||
PRIVATE
|
||||
openvino::runtime::dev
|
||||
)
|
||||
|
||||
#
|
||||
# targets install
|
||||
#
|
||||
ov_install_static_lib(${TARGET_NAME} ${NPU_PLUGIN_COMPONENT})
|
||||
|
||||
ov_developer_package_export_targets(TARGET openvino::npu_logger_utils
|
||||
INSTALL_INCLUDE_DIRECTORIES
|
||||
$<BUILD_INTERFACE:${NPU_UTILS_SOURCE_DIR}/include>)
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "intel_npu/utils/logger/logger.hpp"
|
||||
|
||||
#include <cassert>
|
||||
#include <chrono>
|
||||
#include <cstdarg>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <regex>
|
||||
#include <sstream>
|
||||
|
||||
#define DEFAULT_COLOR "\033[0m"
|
||||
#define RED "\033[31m"
|
||||
#define GREEN "\033[32m"
|
||||
#define YELLOW "\033[33m"
|
||||
#define BLUE "\033[34m"
|
||||
#define CYAN "\033[36m"
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
std::string printFormattedCStr(const char* fmt, ...) {
|
||||
// Process with original buffer
|
||||
const int bufferSize = 256;
|
||||
std::va_list args;
|
||||
va_start(args, fmt);
|
||||
std::va_list argsForFinalBuffer;
|
||||
va_copy(argsForFinalBuffer, args);
|
||||
char buffer[bufferSize];
|
||||
auto requiredBytes = vsnprintf(buffer, bufferSize, fmt, args);
|
||||
va_end(args);
|
||||
|
||||
if (requiredBytes < 0) {
|
||||
va_end(argsForFinalBuffer);
|
||||
return std::string("vsnprintf got error from fmt: ") + fmt;
|
||||
} else if (requiredBytes > bufferSize) {
|
||||
std::string out(requiredBytes, 0); // +1 implicitly
|
||||
vsnprintf(out.data(), requiredBytes + 1, fmt, argsForFinalBuffer);
|
||||
va_end(argsForFinalBuffer);
|
||||
return out;
|
||||
}
|
||||
|
||||
va_end(argsForFinalBuffer);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
//
|
||||
// Logger
|
||||
//
|
||||
static const char* logLevelPrintout[] = {"NONE", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE"};
|
||||
|
||||
Logger& Logger::global() {
|
||||
#if defined(NPU_PLUGIN_DEVELOPER_BUILD) || !defined(NDEBUG)
|
||||
ov::log::Level logLvl = ov::log::Level::WARNING;
|
||||
if (const auto env = std::getenv("OV_NPU_LOG_LEVEL")) {
|
||||
try {
|
||||
std::istringstream is(env);
|
||||
is >> logLvl;
|
||||
} catch (...) {
|
||||
// Use deault log level
|
||||
}
|
||||
}
|
||||
static Logger log("global", logLvl);
|
||||
#else
|
||||
static Logger log("global", ov::log::Level::NO);
|
||||
#endif
|
||||
return log;
|
||||
}
|
||||
|
||||
Logger::Logger(std::string_view name, ov::log::Level lvl) : _name(name), _logLevel(lvl) {}
|
||||
|
||||
Logger Logger::clone(std::string_view name) const {
|
||||
Logger logger(name, level());
|
||||
return logger;
|
||||
}
|
||||
|
||||
bool Logger::isActive(ov::log::Level msgLevel) const {
|
||||
return static_cast<int32_t>(msgLevel) <= static_cast<int32_t>(_logLevel);
|
||||
}
|
||||
|
||||
std::ostream& Logger::getBaseStream() {
|
||||
return std::cout;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
std::ostream& getColor(ov::log::Level msgLevel) {
|
||||
std::ostream& stream = std::cout;
|
||||
|
||||
switch (msgLevel) {
|
||||
case ov::log::Level::ERR:
|
||||
stream << RED;
|
||||
break;
|
||||
case ov::log::Level::WARNING:
|
||||
stream << YELLOW;
|
||||
break;
|
||||
case ov::log::Level::INFO:
|
||||
stream << CYAN;
|
||||
break;
|
||||
case ov::log::Level::DEBUG:
|
||||
stream << GREEN;
|
||||
break;
|
||||
case ov::log::Level::TRACE:
|
||||
stream << BLUE;
|
||||
break;
|
||||
default:
|
||||
stream << DEFAULT_COLOR;
|
||||
}
|
||||
return stream;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::ostream& Logger::getLevelStream(ov::log::Level msgLevel) {
|
||||
return getColor(msgLevel);
|
||||
}
|
||||
|
||||
void Logger::addEntryPackedActive(ov::log::Level msgLevel, std::string_view msg) const {
|
||||
std::stringstream tempStream;
|
||||
char timeStr[] = "undefined_time";
|
||||
time_t now = time(nullptr);
|
||||
struct tm* loctime = localtime(&now);
|
||||
if (loctime != nullptr) {
|
||||
strftime(timeStr, sizeof(timeStr), "%H:%M:%S", loctime);
|
||||
}
|
||||
|
||||
using namespace std::chrono;
|
||||
uint32_t ms = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count() % 1000;
|
||||
auto& stream = getLevelStream(msgLevel);
|
||||
try {
|
||||
tempStream << "[" << logLevelPrintout[static_cast<int32_t>(msgLevel) + 1] << "] " << timeStr << "." << ms
|
||||
<< " [" << _name << "] ";
|
||||
|
||||
tempStream << msg;
|
||||
|
||||
static std::mutex logMtx;
|
||||
std::lock_guard<std::mutex> logMtxLock(logMtx);
|
||||
stream << tempStream.str() << DEFAULT_COLOR;
|
||||
stream << std::endl;
|
||||
stream.flush();
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "Exception caught in Logger::addEntryPackedActive - " << e.what() << std::endl;
|
||||
} catch (...) {
|
||||
std::cerr << "Unknown/internal exception happened in Logger::addEntryPackedActive" << std::endl;
|
||||
}
|
||||
stream.flush();
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set(TARGET_NAME openvino_npu_zero_result_parser)
|
||||
|
||||
file(GLOB_RECURSE SOURCES *.cpp)
|
||||
|
||||
add_library(${TARGET_NAME} STATIC ${SOURCES})
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
ov_add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
|
||||
|
||||
target_include_directories(${TARGET_NAME}
|
||||
PUBLIC
|
||||
$<BUILD_INTERFACE:${NPU_PLUGIN_SOURCE_DIR}/thirdparty/level-zero/include>
|
||||
$<BUILD_INTERFACE:${NPU_PLUGIN_SOURCE_DIR}/thirdparty/level-zero-ext>
|
||||
$<BUILD_INTERFACE:${NPU_UTILS_SOURCE_DIR}/include>
|
||||
)
|
||||
|
||||
#
|
||||
# targets install
|
||||
#
|
||||
ov_install_static_lib(${TARGET_NAME} ${NPU_INTERNAL_COMPONENT})
|
||||
|
|
@ -0,0 +1,328 @@
|
|||
// Copyright (C) 2018-2024 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "intel_npu/utils/zero/zero_result.hpp"
|
||||
|
||||
#include <ze_api.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace intel_npu {
|
||||
|
||||
const std::string ze_result_to_string(const ze_result_t result) {
|
||||
std::string as_string = {};
|
||||
|
||||
switch (result) {
|
||||
case ZE_RESULT_SUCCESS:
|
||||
as_string = "ZE_RESULT_SUCCESS";
|
||||
break;
|
||||
case ZE_RESULT_NOT_READY:
|
||||
as_string = "ZE_RESULT_NOT_READY";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_DEVICE_LOST:
|
||||
as_string = "ZE_RESULT_ERROR_DEVICE_LOST";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY:
|
||||
as_string = "ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY:
|
||||
as_string = "ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_MODULE_BUILD_FAILURE:
|
||||
as_string = "ZE_RESULT_ERROR_MODULE_BUILD_FAILURE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_MODULE_LINK_FAILURE:
|
||||
as_string = "ZE_RESULT_ERROR_MODULE_LINK_FAILURE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_DEVICE_REQUIRES_RESET:
|
||||
as_string = "ZE_RESULT_ERROR_DEVICE_REQUIRES_RESET";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE:
|
||||
as_string = "ZE_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE";
|
||||
break;
|
||||
case ZE_RESULT_EXP_ERROR_DEVICE_IS_NOT_VERTEX:
|
||||
as_string = "ZE_RESULT_EXP_ERROR_DEVICE_IS_NOT_VERTEX";
|
||||
break;
|
||||
case ZE_RESULT_EXP_ERROR_VERTEX_IS_NOT_DEVICE:
|
||||
as_string = "ZE_RESULT_EXP_ERROR_VERTEX_IS_NOT_DEVICE";
|
||||
break;
|
||||
case ZE_RESULT_EXP_ERROR_REMOTE_DEVICE:
|
||||
as_string = "ZE_RESULT_EXP_ERROR_REMOTE_DEVICE";
|
||||
break;
|
||||
// *** Defination not found in enum _ze_result_t level_zero/ze_api.h ***
|
||||
// case ZE_RESULT_EXP_ERROR_OPERANDS_INCOMPATIBLE:
|
||||
// as_string = "ZE_RESULT_EXP_ERROR_OPERANDS_INCOMPATIBLE";
|
||||
// break;
|
||||
// case ZE_RESULT_EXP_RTAS_BUILD_RETRY:
|
||||
// as_string = "ZE_RESULT_EXP_RTAS_BUILD_RETRY";
|
||||
// break;
|
||||
// case ZE_RESULT_EXP_RTAS_BUILD_DEFERRED:
|
||||
// as_string = "ZE_RESULT_EXP_RTAS_BUILD_DEFERRED";
|
||||
// break;
|
||||
case ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS:
|
||||
as_string = "ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_NOT_AVAILABLE:
|
||||
as_string = "ZE_RESULT_ERROR_NOT_AVAILABLE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE:
|
||||
as_string = "ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE";
|
||||
break;
|
||||
case ZE_RESULT_WARNING_DROPPED_DATA:
|
||||
as_string = "ZE_RESULT_WARNING_DROPPED_DATA";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNINITIALIZED:
|
||||
as_string = "ZE_RESULT_ERROR_UNINITIALIZED";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_VERSION:
|
||||
as_string = "ZE_RESULT_ERROR_UNSUPPORTED_VERSION";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_FEATURE:
|
||||
as_string = "ZE_RESULT_ERROR_UNSUPPORTED_FEATURE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_ARGUMENT:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_ARGUMENT";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_NULL_HANDLE:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_NULL_HANDLE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_HANDLE_OBJECT_IN_USE:
|
||||
as_string = "ZE_RESULT_ERROR_HANDLE_OBJECT_IN_USE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_NULL_POINTER:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_NULL_POINTER";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_SIZE:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_SIZE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_SIZE:
|
||||
as_string = "ZE_RESULT_ERROR_UNSUPPORTED_SIZE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_ALIGNMENT:
|
||||
as_string = "ZE_RESULT_ERROR_UNSUPPORTED_ALIGNMENT";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_ENUMERATION:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_ENUMERATION";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_ENUMERATION:
|
||||
as_string = "ZE_RESULT_ERROR_UNSUPPORTED_ENUMERATION";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT:
|
||||
as_string = "ZE_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_NATIVE_BINARY:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_NATIVE_BINARY";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_GLOBAL_NAME:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_GLOBAL_NAME";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_KERNEL_NAME:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_KERNEL_NAME";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_FUNCTION_NAME:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_FUNCTION_NAME";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_MODULE_UNLINKED:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_MODULE_UNLINKED";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_COMMAND_LIST_TYPE:
|
||||
as_string = "ZE_RESULT_ERROR_INVALID_COMMAND_LIST_TYPE";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_OVERLAPPING_REGIONS:
|
||||
as_string = "ZE_RESULT_ERROR_OVERLAPPING_REGIONS";
|
||||
break;
|
||||
case ZE_RESULT_WARNING_ACTION_REQUIRED:
|
||||
as_string = "ZE_RESULT_WARNING_ACTION_REQUIRED";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNKNOWN:
|
||||
as_string = "ZE_RESULT_ERROR_UNKNOWN";
|
||||
break;
|
||||
case ZE_RESULT_FORCE_UINT32:
|
||||
as_string = "ZE_RESULT_FORCE_UINT32";
|
||||
break;
|
||||
default:
|
||||
as_string = "ze_result_t Unrecognized";
|
||||
break;
|
||||
}
|
||||
|
||||
return as_string;
|
||||
}
|
||||
|
||||
const std::string ze_result_to_description(const ze_result_t result) {
|
||||
std::string as_string = {};
|
||||
|
||||
switch (result) {
|
||||
case ZE_RESULT_SUCCESS:
|
||||
as_string = "success";
|
||||
break;
|
||||
case ZE_RESULT_NOT_READY:
|
||||
as_string = "synchronization primitive not signaled";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_DEVICE_LOST:
|
||||
as_string = "device hung, reset, was removed, or driver update occurred";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY:
|
||||
as_string = "insufficient host memory to satisfy call";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY:
|
||||
as_string = "insufficient device memory to satisfy call";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_MODULE_BUILD_FAILURE:
|
||||
as_string = "error occurred when building module, see build log for details";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_MODULE_LINK_FAILURE:
|
||||
as_string = "error occurred when linking modules, see build log for details";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_DEVICE_REQUIRES_RESET:
|
||||
as_string = "device requires a reset";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE:
|
||||
as_string = "device currently in low power state";
|
||||
break;
|
||||
case ZE_RESULT_EXP_ERROR_DEVICE_IS_NOT_VERTEX:
|
||||
as_string = "device is not represented by a fabric vertex";
|
||||
break;
|
||||
case ZE_RESULT_EXP_ERROR_VERTEX_IS_NOT_DEVICE:
|
||||
as_string = "fabric vertex does not represent a device";
|
||||
break;
|
||||
case ZE_RESULT_EXP_ERROR_REMOTE_DEVICE:
|
||||
as_string = "fabric vertex represents a remote device or subdevice";
|
||||
break;
|
||||
// *** Defination not found in enum _ze_result_t level_zero/ze_api.h ***
|
||||
// case ZE_RESULT_EXP_ERROR_OPERANDS_INCOMPATIBLE:
|
||||
// as_string = "operands of comparison are not compatible";
|
||||
// break;
|
||||
// case ZE_RESULT_EXP_RTAS_BUILD_RETRY:
|
||||
// as_string = "ray tracing acceleration structure build operation failed due to insufficient resources, retry "
|
||||
// "with a larger acceleration structure buffer allocation ";
|
||||
// break;
|
||||
// case ZE_RESULT_EXP_RTAS_BUILD_DEFERRED:
|
||||
// as_string = "ray tracing acceleration structure build operation deferred to parallel operation join ";
|
||||
// break;
|
||||
case ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS:
|
||||
as_string = "access denied due to permission level";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_NOT_AVAILABLE:
|
||||
as_string = "resource already in use and simultaneous access not allowed or resource was removed";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE:
|
||||
as_string = "external required dependency is unavailable or missing";
|
||||
break;
|
||||
case ZE_RESULT_WARNING_DROPPED_DATA:
|
||||
as_string = "data may have been dropped";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNINITIALIZED:
|
||||
as_string = "driver is not initialized";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_VERSION:
|
||||
as_string = "generic error code for unsupported versions";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_FEATURE:
|
||||
as_string = "generic error code for unsupported features";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_ARGUMENT:
|
||||
as_string = "generic error code for invalid arguments";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_NULL_HANDLE:
|
||||
as_string = "handle argument is not valid";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_HANDLE_OBJECT_IN_USE:
|
||||
as_string = "object pointed to by handle still in-use by device";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_NULL_POINTER:
|
||||
as_string = "pointer argument may not be nullptr";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_SIZE:
|
||||
as_string = "size argument is invalid (e.g., must not be zero)";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_SIZE:
|
||||
as_string = "size argument is not supported by the device (e.g., too large)";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_ALIGNMENT:
|
||||
as_string = "alignment argument is not supported by the device (e.g., too small)";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT:
|
||||
as_string = "synchronization object in invalid state";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_ENUMERATION:
|
||||
as_string = "enumerator argument is not valid";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_ENUMERATION:
|
||||
as_string = "enumerator argument is not supported by the device";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT:
|
||||
as_string = "image format is not supported by the device";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_NATIVE_BINARY:
|
||||
as_string = "native binary is not supported by the device";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_GLOBAL_NAME:
|
||||
as_string = "global variable is not found in the module";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_KERNEL_NAME:
|
||||
as_string = "kernel name is not found in the module";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_FUNCTION_NAME:
|
||||
as_string = "function name is not found in the module";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION:
|
||||
as_string = "group size dimension is not valid for the kernel or device";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION:
|
||||
as_string = "global width dimension is not valid for the kernel or device";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX:
|
||||
as_string = "kernel argument index is not valid for kernel";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE:
|
||||
as_string = "kernel argument size does not match kernel";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE:
|
||||
as_string = "value of kernel attribute is not valid for the kernel or device";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_MODULE_UNLINKED:
|
||||
as_string = "module with imports needs to be linked before kernels can be created from it";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_INVALID_COMMAND_LIST_TYPE:
|
||||
as_string = "command list type does not match command queue type";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_OVERLAPPING_REGIONS:
|
||||
as_string = "copy operations do not support overlapping regions of memory";
|
||||
break;
|
||||
case ZE_RESULT_WARNING_ACTION_REQUIRED:
|
||||
as_string = "an action is required to complete the desired operation";
|
||||
break;
|
||||
case ZE_RESULT_ERROR_UNKNOWN:
|
||||
as_string = "an action is required to complete the desired operation";
|
||||
break;
|
||||
case ZE_RESULT_FORCE_UINT32:
|
||||
as_string = "FORCE UINT32 (error converting type to uint32)";
|
||||
break;
|
||||
default:
|
||||
as_string = "ze_result_t Unrecognized";
|
||||
break;
|
||||
}
|
||||
|
||||
return as_string;
|
||||
}
|
||||
|
||||
} // namespace intel_npu
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# Copyright (C) 2018-2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set(BUILD_SHARED_LIBS OFF)
|
||||
|
||||
#
|
||||
# zeroApi
|
||||
#
|
||||
|
||||
if(ENABLE_ZEROAPI_BACKEND)
|
||||
# We have to avoid linking against loader with debug postfix due it's a part of driver
|
||||
# and the name will be the same for release and debug configurations
|
||||
set(CMAKE_DEBUG_POSTFIX "")
|
||||
|
||||
# Skip warnings as errors for thirdparty
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
|
||||
string(REPLACE "/WX" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE "/WX" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
# solve pdb access issue
|
||||
set(USE_Z7 ON)
|
||||
# Close spectre for ze loader
|
||||
add_compile_options("/Qspectre-")
|
||||
else()
|
||||
string(REPLACE "-Werror " " " CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
string(REPLACE "-Werror " " " CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
|
||||
-Wno-error=undef \
|
||||
-Wno-error=unused-but-set-variable \
|
||||
-Wno-error=missing-declarations \
|
||||
-Wno-error=stringop-truncation \
|
||||
-Wno-unknown-warning-option")
|
||||
endif()
|
||||
set(CMAKE_COMPILE_WARNING_AS_ERROR OFF)
|
||||
add_subdirectory(level-zero EXCLUDE_FROM_ALL)
|
||||
endif()
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 4ed13f327d3389285592edcf7598ec3cb2bc712e
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 0e1c471356a724ef6d176ba027a68e210d90939e
|
||||
Loading…
Reference in New Issue