diff --git a/src/plugins/intel_npu/src/al/include/sync_infer_request.hpp b/src/plugins/intel_npu/src/al/include/sync_infer_request.hpp index d9a4eaf517f..34e9239e843 100644 --- a/src/plugins/intel_npu/src/al/include/sync_infer_request.hpp +++ b/src/plugins/intel_npu/src/al/include/sync_infer_request.hpp @@ -201,6 +201,7 @@ protected: std::vector _outputAndStateOutputNames; std::unordered_map _nodeNameToLegacyName; + std::unordered_map _legacyNameToNodeName; }; } // namespace intel_npu diff --git a/src/plugins/intel_npu/src/al/src/sync_infer_request.cpp b/src/plugins/intel_npu/src/al/src/sync_infer_request.cpp index 92b9f0e8dda..7a0bfbbb3a6 100644 --- a/src/plugins/intel_npu/src/al/src/sync_infer_request.cpp +++ b/src/plugins/intel_npu/src/al/src/sync_infer_request.cpp @@ -24,11 +24,9 @@ SyncInferRequest::SyncInferRequest(const std::shared_ptr& // Map the node names to the legacy ones used by the I/O tensors in order to allow an easier access to the tensors' // contents - for (const auto& [legacyName, parameterDescriptor] : _metadata.parameters) { - _nodeNameToLegacyName[parameterDescriptor.currentNodeName] = legacyName; - } - for (const auto& [legacyName, resultDescriptor] : _metadata.results) { - _nodeNameToLegacyName[resultDescriptor.currentNodeName] = legacyName; + for (const auto& [name, resultDescriptor] : _metadata.results) { + _nodeNameToLegacyName[name] = resultDescriptor.legacyName; + _legacyNameToNodeName[resultDescriptor.legacyName] = name; } _inputAndStateInputNames = _metadata.inputNames; @@ -48,8 +46,12 @@ SyncInferRequest::SyncInferRequest(const std::shared_ptr& if (contains(_inputAndStateInputNames, shapeName)) { _inputAndStateInputNames.push_back(SHAPE_TENSOR_PREFIX + shapeName); } - if (contains(_outputAndStateOutputNames, shapeName)) { - _outputAndStateOutputNames.push_back(SHAPE_TENSOR_PREFIX + shapeName); + + const auto& shapeNameMatch = _legacyNameToNodeName.find(shapeName); + if (shapeNameMatch != _legacyNameToNodeName.end()) { + if (contains(_outputAndStateOutputNames, shapeNameMatch->second)) { + _outputAndStateOutputNames.push_back(SHAPE_TENSOR_PREFIX + shapeName); + } } } } @@ -83,10 +85,7 @@ std::vector> SyncInferRequest::query_state() const } ov::SoPtr SyncInferRequest::get_tensor(const ov::Output& port) const { - const auto& nodeNameMatch = _nodeNameToLegacyName.find(port.get_node()->get_friendly_name()); - OPENVINO_ASSERT(nodeNameMatch != _nodeNameToLegacyName.end(), "Cannot find tensor for port ", port); - - return _allTensors.at(nodeNameMatch->second); + return _allTensors.at(port.get_node()->get_friendly_name()); } void SyncInferRequest::set_tensor(const ov::Output& port, const ov::SoPtr& tensor) { @@ -97,8 +96,7 @@ void SyncInferRequest::set_tensor(const ov::Output& port, const OPENVINO_THROW("Failed to set tensor. ", ex.what()); } - const std::string& legacyName = _nodeNameToLegacyName.at(port.get_node()->get_friendly_name()); - _allTensors[legacyName] = tensor._ptr; + _allTensors[port.get_node()->get_friendly_name()] = tensor._ptr; } std::vector> SyncInferRequest::get_tensors(const ov::Output& /*port*/) const { @@ -151,14 +149,12 @@ void SyncInferRequest::check_tensor(const ov::Output& port, void SyncInferRequest::check_tensors() const { const auto& inputs = _compiledModel->inputs(); for (size_t i = 0; i < inputs.size(); i++) { - const std::string& legacyName = _nodeNameToLegacyName.at(inputs[i].get_node()->get_friendly_name()); - check_tensor(inputs[i], _allTensors.at(legacyName)); + check_tensor(inputs[i], _allTensors.at(inputs[i].get_node()->get_friendly_name())); } const auto& outputs = _compiledModel->outputs(); for (size_t i = 0; i < outputs.size(); i++) { - const std::string& legacyName = _nodeNameToLegacyName.at(outputs[i].get_node()->get_friendly_name()); - check_tensor(outputs[i], _allTensors.at(legacyName)); + check_tensor(outputs[i], _allTensors.at(outputs[i].get_node()->get_friendly_name())); } } @@ -180,6 +176,7 @@ void SyncInferRequest::allocate_tensor(std::string tensorName, _shapesTensors[tensorName] = tensor; tensorName = SHAPE_TENSOR_PREFIX + tensorName; } + if (tensorType == TensorType::State) { _variableStates[tensorName] = std::make_shared(tensorName, tensor); diff --git a/src/plugins/intel_npu/src/backend/include/zero_executor.hpp b/src/plugins/intel_npu/src/backend/include/zero_executor.hpp index 7ef8d36abcd..e516eb75b82 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_executor.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_executor.hpp @@ -27,7 +27,7 @@ public: ~ZeroExecutor() override; struct ArgumentDescriptor { - ze_graph_argument_properties_t info; + ze_graph_argument_properties_3_t info; uint32_t idx; }; @@ -67,6 +67,7 @@ private: ze_graph_handle_t _graph = nullptr; ze_graph_properties_t _props{}; + std::unordered_map _inputs_desc_map; std::unordered_map _outputs_desc_map; diff --git a/src/plugins/intel_npu/src/backend/include/zero_memory.hpp b/src/plugins/intel_npu/src/backend/include/zero_memory.hpp index 3633ca1a90e..f1935b1ff03 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_memory.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_memory.hpp @@ -96,7 +96,7 @@ private: struct MemoryManagementUnit { MemoryManagementUnit() = default; - void appendArgument(const std::string& name, const ze_graph_argument_properties_t& argument); + void appendArgument(const std::string& name, const std::size_t argSize); /* Allocate Device memories */ void allocate(const ze_device_handle_t device_handle, const ze_context_handle_t context); diff --git a/src/plugins/intel_npu/src/backend/include/zero_utils.hpp b/src/plugins/intel_npu/src/backend/include/zero_utils.hpp index 3a1dd674200..af10fe88fe7 100644 --- a/src/plugins/intel_npu/src/backend/include/zero_utils.hpp +++ b/src/plugins/intel_npu/src/backend/include/zero_utils.hpp @@ -164,7 +164,7 @@ static inline std::size_t layoutCount(const ze_graph_argument_layout_t val) { } } -static inline std::size_t getSizeIOBytes(const ze_graph_argument_properties_t& argument) { +static inline std::size_t getSizeIOBytes(const ze_graph_argument_properties_3_t& argument) { std::size_t num_elements = 1; for (std::size_t i = 0; i < layoutCount(argument.deviceLayout); ++i) { num_elements *= argument.dims[i]; diff --git a/src/plugins/intel_npu/src/backend/src/zero_executor.cpp b/src/plugins/intel_npu/src/backend/src/zero_executor.cpp index 0ce68bf4a0e..468fbf6d95c 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_executor.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_executor.cpp @@ -13,6 +13,7 @@ #include "intel_npu/al/config/common.hpp" #include "intel_npu/al/itt.hpp" +#include "intel_npu/al/prefix.hpp" #include "zero_device.hpp" #include "zero_utils.hpp" @@ -55,11 +56,9 @@ ZeroExecutor::ZeroExecutor(const std::shared_ptr& i _config, _group_ordinal); Fence fence(graph_command_queue, _config); - ze_device_properties_t properties = {}; - properties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; - zeroUtils::throwOnFail("zeDeviceGetProperties", zeDeviceGetProperties(_initStructs->getDevice(), &properties)); OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_GRAPH, itt::domains::LevelZeroBackend, "Executor::ZeroExecutor", "graphCreate"); + ze_graph_desc_t desc{ZE_STRUCTURE_TYPE_GRAPH_DESC_PROPERTIES, nullptr, ZE_GRAPH_FORMAT_NATIVE, @@ -73,17 +72,37 @@ ZeroExecutor::ZeroExecutor(const std::shared_ptr& i OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGetProperties"); zeroUtils::throwOnFail("pfnGetProperties", _graph_ddi_table_ext->pfnGetProperties(_graph, &_props)); - OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGetArgumentProperties"); + auto targetDriverExtVersion = _initStructs->getDriverExtVersion(); + if (targetDriverExtVersion <= ZE_GRAPH_EXT_VERSION_1_1) { + OPENVINO_THROW("Incompatibility between the NPU plugin and driver! The driver version is too old, please " + "update the driver version"); + } + + OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "pfnGetArgumentProperties3"); for (uint32_t index = 0; index < _props.numGraphArgs; ++index) { - ze_graph_argument_properties_t arg; - zeroUtils::throwOnFail("pfnGetArgumentProperties", - _graph_ddi_table_ext->pfnGetArgumentProperties(_graph, index, &arg)); - if (ZE_GRAPH_ARGUMENT_TYPE_INPUT == arg.type) { - _inputs_desc_map.emplace(std::make_pair(std::string(arg.name), ArgumentDescriptor{arg, index})); + ze_graph_argument_properties_3_t arg3; + zeroUtils::throwOnFail("pfnGetArgumentProperties3", + _graph_ddi_table_ext->pfnGetArgumentProperties3(_graph, index, &arg3)); + + if (ZE_GRAPH_ARGUMENT_TYPE_INPUT == arg3.type) { + if (isStateInputName(arg3.name) || isShapeTensorName(arg3.name)) { + _inputs_desc_map.emplace(std::make_pair(std::string(arg3.name), ArgumentDescriptor{arg3, index})); + + } else { + _inputs_desc_map.emplace( + std::make_pair(std::string(arg3.debug_friendly_name), ArgumentDescriptor{arg3, index})); + } } else { - _outputs_desc_map.emplace(std::make_pair(std::string(arg.name), ArgumentDescriptor{arg, index})); + if (isStateOutputName(arg3.name) || isShapeTensorName(arg3.name)) { + _outputs_desc_map.emplace(std::make_pair(std::string(arg3.name), ArgumentDescriptor{arg3, index})); + + } else { + _outputs_desc_map.emplace( + std::make_pair(std::string(arg3.debug_friendly_name), ArgumentDescriptor{arg3, index})); + } } } + OV_ITT_TASK_NEXT(ZERO_EXECUTOR_GRAPH, "appendGraphInitialize"); graph_command_list.appendGraphInitialize(_graph); graph_command_list.close(); diff --git a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp index 4e73dcf1244..89445ebcaf1 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp @@ -91,6 +91,8 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr& return std::find(container.begin(), container.end(), value) != container.end(); }; + auto allocator = zeroMemory::HostMemAllocator(backendPtr); + for (const std::string& inputName : _metadata.inputNames) { if (!executorInputDescriptors.count(inputName)) { OPENVINO_THROW("Invalid graph input descriptor key: " + inputName); @@ -99,15 +101,15 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr& const IONodeDescriptor& parameterDescriptor = _metadata.parameters.at(inputName); check_level_zero_attributes_match(parameterDescriptor, executorInputDescriptors.at(inputName), inputName); - ov::Allocator allocator; + ov::Allocator inputAllocator; if (properties.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) { - allocator = zeroMemory::HostMemAllocator(backendPtr, ZE_HOST_MEM_ALLOC_FLAG_BIAS_WRITE_COMBINED); + inputAllocator = zeroMemory::HostMemAllocator(backendPtr, ZE_HOST_MEM_ALLOC_FLAG_BIAS_WRITE_COMBINED); } else { - allocator = zeroMemory::HostMemAllocator(backendPtr); - } + inputAllocator = zeroMemory::HostMemAllocator(backendPtr); + }; // The I/O buffers already allocated using the Level Zero API are being reused here - allocate_tensor(inputName, parameterDescriptor, TensorType::InputOrOutput, allocator); + allocate_tensor(inputName, parameterDescriptor, TensorType::InputOrOutput, inputAllocator); if (contains(_metadata.shapeNames, inputName)) { const std::string shapeBufferName = SHAPE_TENSOR_PREFIX + inputName; @@ -117,8 +119,7 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr& executorInputDescriptors.at(shapeBufferName), shapeBufferName); - auto allocator = zeroMemory::HostMemAllocator(backendPtr); - allocate_tensor(inputName, shapeDescriptor, TensorType::Shape, allocator); + allocate_tensor(inputName, shapeDescriptor, TensorType::Shape, inputAllocator); } } @@ -130,20 +131,20 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr& const IONodeDescriptor& resultDescriptor = _metadata.results.at(outputName); check_level_zero_attributes_match(resultDescriptor, executorOutputDescriptors.at(outputName), outputName); - auto allocator = zeroMemory::HostMemAllocator(backendPtr); - allocate_tensor(outputName, resultDescriptor, TensorType::InputOrOutput, allocator); - if (contains(_metadata.shapeNames, outputName)) { - const std::string shapeBufferName = SHAPE_TENSOR_PREFIX + outputName; - const IONodeDescriptor& shapeDescriptor = _metadata.shapes.at(outputName); + const auto& shapeNameMatch = _nodeNameToLegacyName.find(outputName); + if (shapeNameMatch != _nodeNameToLegacyName.end()) { + if (contains(_metadata.shapeNames, shapeNameMatch->second)) { + const std::string shapeBufferName = SHAPE_TENSOR_PREFIX + shapeNameMatch->second; + const IONodeDescriptor& shapeDescriptor = _metadata.shapes.at(shapeNameMatch->second); - check_level_zero_attributes_match(shapeDescriptor, - executorOutputDescriptors.at(shapeBufferName), - shapeBufferName); + check_level_zero_attributes_match(shapeDescriptor, + executorOutputDescriptors.at(shapeBufferName), + shapeBufferName); - auto allocator = zeroMemory::HostMemAllocator(backendPtr); - allocate_tensor(outputName, shapeDescriptor, TensorType::Shape, allocator); + allocate_tensor(shapeNameMatch->second, shapeDescriptor, TensorType::Shape, allocator); + } } } @@ -166,8 +167,6 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr& executorOutputDescriptors.at(stateOutputBufferName), stateOutputBufferName); - auto allocator = zeroMemory::HostMemAllocator(backendPtr); - // Only one buffer per state variable is required, we'll use the "output" one since this one captures the latest // tensor value allocate_tensor(stateName, stateDescriptor, TensorType::State, allocator); @@ -226,15 +225,18 @@ void ZeroInferRequest::get_result() { if (isShapeTensorName(name)) { const auto actualTensorName = name.substr(SHAPE_TENSOR_PREFIX.size()); - ov::Shape actualDims; - actualDims.reserve(outputTensor->get_size()); + const auto& shapeNameMatch = _legacyNameToNodeName.find(actualTensorName); + if (shapeNameMatch != _legacyNameToNodeName.end()) { + ov::Shape actualDims; + actualDims.reserve(outputTensor->get_size()); - for (size_t i = 0; i < outputTensor->get_size(); ++i) { - const auto reverseIdx = outputTensor->get_size() - 1 - i; - actualDims.push_back(outputTensor->data()[reverseIdx]); + for (size_t i = 0; i < outputTensor->get_size(); ++i) { + const auto reverseIdx = outputTensor->get_size() - 1 - i; + actualDims.push_back(outputTensor->data()[reverseIdx]); + } + auto& tensorToBeReshaped = _allTensors.at(shapeNameMatch->second); + tensorToBeReshaped->set_shape(actualDims); } - auto& tensorToBeReshaped = _allTensors.at(actualTensorName); - tensorToBeReshaped->set_shape(actualDims); } uint8_t* tensorBuffer = reinterpret_cast(outputTensor->data()); diff --git a/src/plugins/intel_npu/src/backend/src/zero_memory.cpp b/src/plugins/intel_npu/src/backend/src/zero_memory.cpp index d92e817f1cd..15bbe9911c2 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_memory.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_memory.cpp @@ -69,10 +69,9 @@ bool HostMemAllocator::is_equal(const HostMemAllocator& other) const { return other._data != nullptr && _data != nullptr && other._data == _data; } -void MemoryManagementUnit::appendArgument(const std::string& name, const ze_graph_argument_properties_t& argument) { +void MemoryManagementUnit::appendArgument(const std::string& name, const std::size_t argSize) { _offsets.emplace(std::make_pair(name, _size)); - const std::size_t argSize = zeroUtils::getSizeIOBytes(argument); _size += argSize + alignment - (argSize % alignment); // is this really necessary? if 0==argSize%alignment -> add 1 * alignment } diff --git a/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp b/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp index 531b3a087a2..c79197d79e9 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp @@ -41,7 +41,7 @@ public: OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Zero_infer_request::DiscretePipeline::DiscretePipeline"); for (const auto& desc : executor->inputs_desc_map()) { - _deviceInputs.appendArgument(desc.first, desc.second.info); + _deviceInputs.appendArgument(desc.first, zeroUtils::getSizeIOBytes(desc.second.info)); } _deviceInputs.allocate(device_handle, context); @@ -61,7 +61,7 @@ public: _event[stage::UPLOAD].AppendSignalEvent(_command_list[stage::UPLOAD]); for (const auto& desc : executor->outputs_desc_map()) { - _deviceOutputs.appendArgument(desc.first, desc.second.info); + _deviceOutputs.appendArgument(desc.first, zeroUtils::getSizeIOBytes(desc.second.info)); } _deviceOutputs.allocate(device_handle, context); diff --git a/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp b/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp index 7746d97c5c6..ff7b51443ac 100644 --- a/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp +++ b/src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp @@ -996,8 +996,8 @@ static void getNodeDescriptor(IONodeDescriptorMap& nodeDescriptors, } const std::string& legacyName = arg.name; - names.push_back(legacyName); - nodeDescriptors[legacyName] = + names.push_back(arg.debug_friendly_name); + nodeDescriptors[arg.debug_friendly_name] = {legacyName, arg.debug_friendly_name, std::move(outputTensorNames), precision, shape, shape}; }