natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[NPU] Adding support for handling batching on the plugin (#23915) 

### Details:
- *It adds support for handling batching on the plugin side only for
integrated platforms*
- *Handling batching by plugin means splitting batch size on the NPU
plugin side and using concurrent inferences for each size*
- *With this PR batching on the plugin is supported only when it is
found on the 0-th dimension and stateful models are also not supported
with batching on plugin*
- *BATCH_MODE is an internal property, we can choose which in which mode
we prefer to handle the batching. If AUTO is set it will try to handle
it on the plugin side but will fallback on the compiler if it doesn't
meet all the conditions for plugin batching. In the case of PLUGIN, it
will try to compile the networks(find batch and force it to 1 for the
compiler) for plugin batching. Otherwise, if COMPILER batching is used
it will work as until now. No other changes*

Diagram of the process flow:
![Screenshot 2024-03-14
172714](https://github.com/openvinotoolkit/openvino/assets/10560145/bf4a00bb-67a1-4c49-af94-5edaa37e9d57)

### Tickets:
 - *[E#103116]*
This commit is contained in:
Bogdan Pereanu 2024-04-29 10:40:04 +03:00 committed by GitHub
parent 22d869dd97
commit 057f41c138
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 565 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/plugins/intel_npu/src/al/include/intel_npu/al/config/common.hpp
View File

@ -240,6 +240,23 @@ struct INTERNAL_SUPPORTED_PROPERTIES final : OptionBase<INTERNAL_SUPPORTED_PROPE
}
};
//
// BATCH_MODE
//
struct BATCH_MODE final : OptionBase<BATCH_MODE, ov::intel_npu::BatchMode> {
static std::string_view key() {
return ov::intel_npu::batch_mode.name();
}
static constexpr std::string_view getTypeName() {
return "ov::intel_npu::BatchMode";
}
static ov::intel_npu::BatchMode parse(std::string_view val);
static std::string toString(const ov::intel_npu::BatchMode& val);
};
} // namespace intel_npu
namespace ov {

2
src/plugins/intel_npu/src/al/include/npu.hpp
View File

@ -33,6 +33,8 @@ public:
virtual uint32_t getDriverExtVersion() const;
/** @brief Get name of backend */
virtual const std::string getName() const = 0;
/** @brief Backend has support for concurrency batching */
virtual bool isBatchingSupported() const = 0;
/** @brief Register backend-specific options */
virtual void registerOptions(OptionsDesc& options) const;

45
src/plugins/intel_npu/src/al/include/npu_private_properties.hpp
View File

@ -151,6 +151,43 @@ inline std::ostream& operator<<(std::ostream& out, const ElfCompilerBackend& fmt
return out;
}
/**
* @brief [Only for NPU Plugin]
* Type: String. Default is "AUTO".
* This option is added for enabling batching on plugin.
* Possible values: "AUTO", "COMPILER", "PLUGIN".
*/
enum class BatchMode {
AUTO = 0,
COMPILER = 1,
PLUGIN = 2,
};
/**
* @brief Prints a string representation of ov::intel_npu::BatchMode to a stream
* @param out An output stream to send to
* @param fmt A value for batching on plugin to print to a stream
* @return A reference to the `out` stream
* @note Configuration API v 2.0
*/
inline std::ostream& operator<<(std::ostream& out, const BatchMode& fmt) {
switch (fmt) {
case BatchMode::AUTO: {
out << "AUTO";
} break;
case BatchMode::COMPILER: {
out << "COMPILER";
} break;
case BatchMode::PLUGIN: {
out << "PLUGIN";
} break;
default:
out << static_cast<uint32_t>(fmt);
break;
}
return out;
}
/**
* @brief [Only for NPU Plugin]
* Type: string, default is MODEL.
@ -321,6 +358,14 @@ static constexpr ov::Property<ProfilingType> profiling_type{"NPU_PROFILING_TYPE"
*/
static constexpr ov::Property<ElfCompilerBackend> use_elf_compiler_backend{"NPU_USE_ELF_COMPILER_BACKEND"};
/**
* @brief [Only for NPU Plugin]
* Type: String. Default is "AUTO".
* This option is added for enabling batching on plugin, otherwise batching will be handled by compiler.
* Possible values: "AUTO", "PLUGIN", "COMPILER".
*/
static constexpr ov::Property<BatchMode> batch_mode{"NPU_BATCH_MODE"};
/**
* @brief [Only for NPU Plugin]
* Type: integer, default is 1

25
src/plugins/intel_npu/src/al/src/config/common.cpp
View File

@ -21,6 +21,7 @@ void intel_npu::registerCommonOptions(OptionsDesc& desc) {
desc.add<DEVICE_ID>();
desc.add<CACHE_DIR>();
desc.add<LOADED_FROM_CACHE>();
desc.add<BATCH_MODE>();
}
//
@ -53,3 +54,27 @@ ov::hint::PerformanceMode intel_npu::PERFORMANCE_HINT::parse(std::string_view va
OPENVINO_THROW("Value '", val, "' is not a valid PERFORMANCE_HINT option");
}
//
// BATCH_MODE
//
ov::intel_npu::BatchMode intel_npu::BATCH_MODE::parse(std::string_view val) {
if (val == "AUTO") {
return ov::intel_npu::BatchMode::AUTO;
} else if (val == "COMPILER") {
return ov::intel_npu::BatchMode::COMPILER;
} else if (val == "PLUGIN") {
return ov::intel_npu::BatchMode::PLUGIN;
}
OPENVINO_THROW("Value '", val, "'is not a valid BATCH_MODE option");
}
std::string intel_npu::BATCH_MODE::toString(const ov::intel_npu::BatchMode& val) {
std::stringstream strStream;
strStream << val;
return strStream.str();
}

2
src/plugins/intel_npu/src/backend/include/zero_backend.hpp
View File

@ -24,6 +24,8 @@ public:
uint32_t getDriverVersion() const override;
uint32_t getDriverExtVersion() const override;
bool isBatchingSupported() const override;
private:
std::shared_ptr<ZeroInitStructsHolder> _instance;

14
src/plugins/intel_npu/src/backend/include/zero_infer_request.hpp
View File

@ -17,6 +17,10 @@
#include "zero_utils.hpp"
#include "zero_wrappers.hpp"
namespace {
constexpr std::size_t DEFAULT_BATCH_SIZE = 1;
} // namespace
namespace intel_npu {
class ZeroInferRequest final : public SyncInferRequest {
@ -42,10 +46,14 @@ private:
const Config _config;
Logger _logger;
zeroProfiling::ProfilingPool _profiling_pool;
zeroProfiling::ProfilingQuery _profiling_query;
std::shared_ptr<zeroProfiling::NpuInferProfiling> _npu_profiling;
zeroProfiling::ProfilingPool _profilingPool;
zeroProfiling::ProfilingQuery _profilingQuery;
std::shared_ptr<zeroProfiling::NpuInferProfiling> _npuProfiling;
std::unique_ptr<Pipeline> _pipeline;
// If batching is handled on the compiler side then batching on the plugin shall be set to 1, we don't do any
// specific operations on the plugin in this case.
size_t _batchSize = DEFAULT_BATCH_SIZE;
};
} // namespace intel_npu

12
src/plugins/intel_npu/src/backend/include/zero_memory.hpp
View File

@ -88,16 +88,16 @@ private:
static const std::size_t _alignment = STANDARD_PAGE_SIZE;
};
// For graph arguments (inputs and outputs) memory should be located on a host side. For discrete HW
// generation the arguments has to be moved to device side to make it accessible.
// MemoryManagementUnit allow to keeps device allocations in case of discrete HW.
// Usage: we should append graph arguments with corresponding names with `appendArgument` call
// to prepare size statistics and lookup table. To commit memory allocation we should call `allocate`
// Graph arguments (inputs and outputs) need to be allocated in the host memory.
// For discrete platforms, graph arguments need to be copied into the device memory.
// MemoryMangementUnit is used to allocate memory in the device memory.
// Usage: we should append graph arguments with corresponding names with `appendArgument` call to prepare size
// statistics and lookup table. To commit memory allocation we should call `allocate`
struct MemoryManagementUnit {
MemoryManagementUnit() = default;
void appendArgument(const std::string& name, const std::size_t argSize);
/* Allocate Device memories */
void allocate(const ze_device_handle_t device_handle, const ze_context_handle_t context);
std::size_t getSize() const;

9
src/plugins/intel_npu/src/backend/include/zero_pipeline.hpp
View File

@ -20,9 +20,9 @@ public:
Pipeline& operator=(Pipeline&&) = delete;
virtual ~Pipeline() = default;
virtual void push() = 0;
virtual void pull() = 0;
virtual void reset() const = 0;
virtual void push(size_t batch_index) = 0;
virtual void pull(size_t batch_index) = 0;
virtual void reset(size_t batch_index) const = 0;
protected:
zeroMemory::MemoryManagementUnit _deviceInputs;
@ -34,5 +34,6 @@ std::unique_ptr<Pipeline> makePipeline(const std::shared_ptr<const IExecutor>& e
zeroProfiling::ProfilingPool& profiling_pool,
zeroProfiling::ProfilingQuery& profiling_query,
std::shared_ptr<zeroProfiling::NpuInferProfiling> npu_profiling,
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors);
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors,
const size_t batch_size);
} // namespace intel_npu

4
src/plugins/intel_npu/src/backend/src/zero_backend.cpp
View File

@ -28,6 +28,10 @@ uint32_t ZeroEngineBackend::getDriverExtVersion() const {
return _instance->getDriverExtVersion();
}
bool ZeroEngineBackend::isBatchingSupported() const {
return _instance->getDriverExtVersion() >= ZE_GRAPH_EXT_VERSION_1_6;
}
ZeroEngineBackend::~ZeroEngineBackend() = default;
const std::shared_ptr<IDevice> ZeroEngineBackend::getDevice() const {

207
src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp
View File

@ -15,6 +15,8 @@ using namespace intel_npu;
namespace {
constexpr std::size_t BATCH_AXIS = 0;
/**
* @brief Checks that the metadata of the provided descriptor corresponds to the values registered in the Level Zero
* structure.
@ -22,9 +24,9 @@ namespace {
* @param zeDescriptor The Level Zero specific structure used for comparison.
* @param name Tensor identifier used for error logging.
*/
void check_level_zero_attributes_match(const IONodeDescriptor& nodeDescriptor,
const ZeroExecutor::ArgumentDescriptor& zeDescriptor,
const std::string& name) {
void checkLevelZeroAttributesMatch(const IONodeDescriptor& nodeDescriptor,
const ZeroExecutor::ArgumentDescriptor& zeDescriptor,
const std::string& name) {
const ov::element::Type_t ovPrecision = nodeDescriptor.precision;
const ze_graph_argument_precision_t zePrecision = zeDescriptor.info.devicePrecision;
@ -32,7 +34,7 @@ void check_level_zero_attributes_match(const IONodeDescriptor& nodeDescriptor,
OPENVINO_THROW("Precision mismatch for parameter " + name);
}
const std::vector<size_t>& ovDimensions = nodeDescriptor.originalShape.get_max_shape();
const std::vector<size_t>& ovDimensions = nodeDescriptor.transposedShape.get_max_shape();
if (ovDimensions.size() > ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE) {
OPENVINO_THROW(
@ -40,16 +42,99 @@ void check_level_zero_attributes_match(const IONodeDescriptor& nodeDescriptor,
"Given: " + std::to_string(ovDimensions.size()));
}
for (size_t index = 0; index < ovDimensions.size(); ++index) {
if (ovDimensions[index] != zeDescriptor.info.dims[index] && !nodeDescriptor.originalShape.is_dynamic()) {
OPENVINO_THROW("Shape mismatch for parameter " + name);
}
}
for (size_t index = ovDimensions.size(); index < ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE; ++index) {
if (zeDescriptor.info.dims[index] != 0 && zeDescriptor.info.dims[index] != 1) {
OPENVINO_THROW("Shape mismatch for parameter " + name);
}
}
for (size_t index = 1; index < ovDimensions.size(); ++index) {
if (ovDimensions[index] != zeDescriptor.info.dims[index] && !nodeDescriptor.transposedShape.is_dynamic()) {
OPENVINO_THROW("Shape mismatch for parameter " + name);
}
}
}
std::optional<size_t> getBatchSizeForNode(const IONodeDescriptor& nodeDescriptor,
const ZeroExecutor::ArgumentDescriptor& zeDescriptor) {
Logger logger("GetBatchSizeForNode", Logger::global().level());
const std::vector<size_t>& ovDimensions = nodeDescriptor.originalShape.get_shape();
switch (zeDescriptor.info.deviceLayout) {
case ZE_GRAPH_ARGUMENT_LAYOUT_NCHW:
case ZE_GRAPH_ARGUMENT_LAYOUT_NHWC:
case ZE_GRAPH_ARGUMENT_LAYOUT_NCDHW:
case ZE_GRAPH_ARGUMENT_LAYOUT_NDHWC:
case ZE_GRAPH_ARGUMENT_LAYOUT_NC:
if ((ovDimensions[BATCH_AXIS] == zeDescriptor.info.dims[BATCH_AXIS]) &&
(ovDimensions[BATCH_AXIS] != DEFAULT_BATCH_SIZE)) {
logger.info("Batching on the plugin is not used, batching is handled by the compiler");
return std::nullopt;
} else {
return ovDimensions[BATCH_AXIS];
}
break;
default:
logger.info("Batching on the plugin is working only when batching is found on 0th dimension");
return std::nullopt;
}
return DEFAULT_BATCH_SIZE;
}
/**
* @brief Get the batch size to be handled on the plugin.
* @details Analyze the shape from the compiled model with the shape from the originalShape and get the originalShape if
* it is different.
* @param metadata A map to represent descriptions for inputs and outputs of a network.
* @param executorInputDescriptors A map to represent Level zero inputs descriptors.
* @param executorOutputDescriptors A map to represent Level zero outputs descriptors.
*/
std::optional<size_t> getBatchSize(
const NetworkMetadata& metadata,
const std::unordered_map<std::string, ZeroExecutor::ArgumentDescriptor>& executorInputDescriptors,
const std::unordered_map<std::string, ZeroExecutor::ArgumentDescriptor>& executorOutputDescriptors) {
std::set<size_t> batch_size;
Logger logger("getBatchSize", Logger::global().level());
for (const std::string& inputName : metadata.inputNames) {
auto batchSizeForNode =
getBatchSizeForNode(metadata.parameters.at(inputName), executorInputDescriptors.at(inputName));
if (batchSizeForNode.has_value()) {
batch_size.insert(*batchSizeForNode);
} else {
return std::nullopt;
}
}
for (const std::string& outputName : metadata.outputNames) {
if (!executorOutputDescriptors.count(outputName)) {
OPENVINO_THROW("Invalid graph output descriptor key: " + outputName);
}
auto batchSizeForNode =
getBatchSizeForNode(metadata.results.at(outputName), executorOutputDescriptors.at(outputName));
if (batchSizeForNode.has_value()) {
batch_size.insert(*batchSizeForNode);
} else {
return std::nullopt;
}
}
if (batch_size.size() != 1) {
logger.info("Batching works only when we have the same batch size for all tensors!");
return std::nullopt;
}
auto it = batch_size.begin();
if (*it) {
return *it;
}
return std::nullopt;
}
} // namespace
@ -64,12 +149,10 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>&
_executor(static_cast<const ZeroExecutor*>(_executorPtr.get())),
_config(config),
_logger("ZeroInferRequest", config.get<LOG_LEVEL>()),
_profiling_pool(_executor->graph(),
zeroProfiling::POOL_SIZE,
_executor->getInitStructs()->getProfilingDdiTable()),
_profiling_query(0,
_executor->getInitStructs()->getDevice(),
_executor->getInitStructs()->getProfilingDdiTable()) {
_profilingPool(_executor->graph(), zeroProfiling::POOL_SIZE, _executor->getInitStructs()->getProfilingDdiTable()),
_profilingQuery(0,
_executor->getInitStructs()->getDevice(),
_executor->getInitStructs()->getProfilingDdiTable()) {
const std::unordered_map<std::string, ZeroExecutor::ArgumentDescriptor>& executorInputDescriptors =
_executor->inputs_desc_map();
const std::unordered_map<std::string, ZeroExecutor::ArgumentDescriptor>& executorOutputDescriptors =
@ -77,9 +160,9 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>&
auto proftype = config.get<PROFILING_TYPE>();
if (proftype == ov::intel_npu::ProfilingType::INFER) {
_npu_profiling = std::make_shared<zeroProfiling::NpuInferProfiling>(_executor->getInitStructs()->getContext(),
_executor->getInitStructs()->getDevice(),
_config.get<LOG_LEVEL>());
_npuProfiling = std::make_shared<zeroProfiling::NpuInferProfiling>(_executor->getInitStructs()->getContext(),
_executor->getInitStructs()->getDevice(),
_config.get<LOG_LEVEL>());
}
ze_device_properties_t properties = {};
@ -97,9 +180,25 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>&
if (!executorInputDescriptors.count(inputName)) {
OPENVINO_THROW("Invalid graph input descriptor key: " + inputName);
}
}
const IONodeDescriptor& parameterDescriptor = _metadata.parameters.at(inputName);
check_level_zero_attributes_match(parameterDescriptor, executorInputDescriptors.at(inputName), inputName);
for (const std::string& outputName : _metadata.outputNames) {
if (!executorOutputDescriptors.count(outputName)) {
OPENVINO_THROW("Invalid graph output descriptor key: " + outputName);
}
}
if (config.get<BATCH_MODE>() != ov::intel_npu::BatchMode::COMPILER) {
auto batchSize = getBatchSize(_metadata, executorInputDescriptors, executorOutputDescriptors);
if (batchSize.has_value()) {
_batchSize = *batchSize;
}
}
for (const std::string& inputName : _metadata.inputNames) {
IONodeDescriptor& parameterDescriptor = _metadata.parameters.at(inputName);
checkLevelZeroAttributesMatch(parameterDescriptor, executorInputDescriptors.at(inputName), inputName);
ov::Allocator inputAllocator;
if (properties.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED) {
@ -108,6 +207,12 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>&
inputAllocator = zeroMemory::HostMemAllocator(backendPtr);
};
// When batching is handled by the plugin we need to modify transposed shape with the original batch size since
// it will be forced to 1 at the compilation time
if (_batchSize > DEFAULT_BATCH_SIZE) {
parameterDescriptor.transposedShape[BATCH_AXIS] = _batchSize;
}
// The I/O buffers already allocated using the Level Zero API are being reused here
allocate_tensor(inputName, parameterDescriptor, TensorType::InputOrOutput, inputAllocator);
@ -115,21 +220,23 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>&
const std::string shapeBufferName = SHAPE_TENSOR_PREFIX + inputName;
const IONodeDescriptor& shapeDescriptor = _metadata.shapes.at(inputName);
check_level_zero_attributes_match(shapeDescriptor,
executorInputDescriptors.at(shapeBufferName),
shapeBufferName);
checkLevelZeroAttributesMatch(shapeDescriptor,
executorInputDescriptors.at(shapeBufferName),
shapeBufferName);
allocate_tensor(inputName, shapeDescriptor, TensorType::Shape, inputAllocator);
}
}
for (const std::string& outputName : _metadata.outputNames) {
if (!executorOutputDescriptors.count(outputName)) {
OPENVINO_THROW("Invalid graph output descriptor key: " + outputName);
}
IONodeDescriptor& resultDescriptor = _metadata.results.at(outputName);
checkLevelZeroAttributesMatch(resultDescriptor, executorOutputDescriptors.at(outputName), outputName);
const IONodeDescriptor& resultDescriptor = _metadata.results.at(outputName);
check_level_zero_attributes_match(resultDescriptor, executorOutputDescriptors.at(outputName), outputName);
// When batching is handled by the plugin we need to modify transposed shape with the original batch size since
// it will be forced to 1 at the compilation time
if (_batchSize > DEFAULT_BATCH_SIZE) {
resultDescriptor.transposedShape[BATCH_AXIS] = _batchSize;
}
allocate_tensor(outputName, resultDescriptor, TensorType::InputOrOutput, allocator);
@ -139,9 +246,9 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>&
const std::string shapeBufferName = SHAPE_TENSOR_PREFIX + shapeNameMatch->second;
const IONodeDescriptor& shapeDescriptor = _metadata.shapes.at(shapeNameMatch->second);
check_level_zero_attributes_match(shapeDescriptor,
executorOutputDescriptors.at(shapeBufferName),
shapeBufferName);
checkLevelZeroAttributesMatch(shapeDescriptor,
executorOutputDescriptors.at(shapeBufferName),
shapeBufferName);
allocate_tensor(shapeNameMatch->second, shapeDescriptor, TensorType::Shape, allocator);
}
@ -160,12 +267,12 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>&
}
const IONodeDescriptor& stateDescriptor = _metadata.states.at(stateName);
check_level_zero_attributes_match(stateDescriptor,
executorInputDescriptors.at(stateInputBufferName),
stateInputBufferName);
check_level_zero_attributes_match(stateDescriptor,
executorOutputDescriptors.at(stateOutputBufferName),
stateOutputBufferName);
checkLevelZeroAttributesMatch(stateDescriptor,
executorInputDescriptors.at(stateInputBufferName),
stateInputBufferName);
checkLevelZeroAttributesMatch(stateDescriptor,
executorOutputDescriptors.at(stateOutputBufferName),
stateOutputBufferName);
// Only one buffer per state variable is required, we'll use the "output" one since this one captures the latest
// tensor value
@ -173,7 +280,13 @@ ZeroInferRequest::ZeroInferRequest(const std::shared_ptr<ZeroInitStructsHolder>&
}
/// Construct pipepline
_pipeline = makePipeline(_executorPtr, _config, _profiling_pool, _profiling_query, _npu_profiling, _copyAllTensors);
_pipeline = makePipeline(_executorPtr,
_config,
_profilingPool,
_profilingQuery,
_npuProfiling,
_copyAllTensors,
_batchSize);
}
void ZeroInferRequest::infer() {
@ -211,13 +324,17 @@ void ZeroInferRequest::infer_async() {
}
}
_pipeline->push();
for (size_t i = 0; i < _batchSize; i++) {
_pipeline->push(i);
}
}
void ZeroInferRequest::get_result() {
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "get_result");
_pipeline->pull();
for (size_t i = 0; i < _batchSize; i++) {
_pipeline->pull(i);
}
for (const auto& name : _outputAndStateOutputNames) {
const auto& outputTensor = _allTensors.at(name);
@ -251,7 +368,9 @@ void ZeroInferRequest::get_result() {
}
}
_pipeline->reset();
for (size_t i = 0; i < _batchSize; i++) {
_pipeline->reset(i);
}
_logger.debug("InferRequest::get_result finished");
}
@ -302,13 +421,13 @@ std::vector<ov::ProfilingInfo> ZeroInferRequest::get_profiling_info() const {
} else {
auto proftype = _config.get<PROFILING_TYPE>();
if (proftype == ov::intel_npu::ProfilingType::INFER) {
return _npu_profiling->getNpuInferStatistics();
return _npuProfiling->getNpuInferStatistics();
} else { /// proftype = MODEL or undefined = fallback to model profiling
return _profiling_query.getLayerStatistics();
return _profilingQuery.getLayerStatistics();
}
}
}
std::vector<uint8_t> ZeroInferRequest::get_raw_profiling_data() const {
return _profiling_query.getData<uint8_t>();
return _profilingQuery.getData<uint8_t>();
}

109
src/plugins/intel_npu/src/backend/src/zero_pipeline.cpp
View File

@ -94,7 +94,7 @@ public:
DiscretePipeline& operator=(const DiscretePipeline&) = delete;
virtual ~DiscretePipeline() = default;
void push() override {
void push(size_t) override {
OV_ITT_TASK_CHAIN(ZERO_INFER_REQUEST_DP_PUSH,
itt::domains::LevelZeroBackend,
"DiscretePipeline::push",
@ -107,7 +107,7 @@ public:
_command_queues[stage::EXECUTE]->executeCommandList(_command_list[stage::EXECUTE], _fence[stage::EXECUTE]);
};
void pull() override {
void pull(size_t) override {
OV_ITT_TASK_CHAIN(ZERO_INFER_REQUEST_DP_PULL,
itt::domains::LevelZeroBackend,
"DiscretePipeline::pull",
@ -122,7 +122,7 @@ public:
_fence[stage::READBACK].hostSynchronize();
};
void reset() const override {
void reset(size_t) const override {
// Reset the fence objects
for (auto& fence : _fence) {
fence.reset();
@ -149,70 +149,87 @@ public:
std::shared_ptr<zeroProfiling::NpuInferProfiling> npu_profiling,
CommandQueue& command_queue,
const uint32_t& group_ordinal,
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors)
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors,
const size_t batch_size)
: _config(config),
_command_queue{command_queue},
_command_list{device_handle, context, graph_ddi_table_ext, _config, group_ordinal},
_fence{_command_queue, _config},
_event_pool{device_handle, context, 1, _config},
_event{_event_pool.handle(), 0, _config},
_event_pool{device_handle, context, batch_size ? static_cast<uint32_t>(batch_size) : 1, _config},
_npu_profiling(npu_profiling) {
const ZeroExecutor* executor = static_cast<const ZeroExecutor*>(executorPtr.get());
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend,
"Zero_infer_request::IntegratedPipeline::IntegratedPipeline");
for (const auto& desc : executor->inputs_desc_map()) {
const std::shared_ptr<ov::ITensor>& inputTensor = tensors.at(desc.first);
executor->setArgumentValue(desc.second.idx, inputTensor->data());
_command_lists.reserve(batch_size);
_events.reserve(batch_size);
_fences.reserve(batch_size);
for (size_t i = 0; i < batch_size; i++) {
_command_lists.emplace_back(
std::make_unique<CommandList>(device_handle, context, graph_ddi_table_ext, _config, group_ordinal));
_events.emplace_back(std::make_unique<Event>(_event_pool.handle(), static_cast<uint32_t>(i), _config));
_fences.emplace_back(std::make_unique<Fence>(_command_queue, _config));
}
for (const auto& desc : executor->outputs_desc_map()) {
const std::shared_ptr<ov::ITensor>& outputTensor = tensors.at(desc.first);
executor->setArgumentValue(desc.second.idx, outputTensor->data());
}
for (size_t i = 0; i < batch_size; i++) {
for (const auto& desc : executor->inputs_desc_map()) {
const std::shared_ptr<ov::ITensor>& inputTensor = tensors.at(desc.first);
size_t inputTensorByteSize = inputTensor->get_byte_size();
executor->setArgumentValue(
desc.second.idx,
static_cast<unsigned char*>(inputTensor->data()) + (i * inputTensorByteSize) / batch_size);
}
/// append timestamp command if feature was activated
if (_npu_profiling != nullptr) {
_command_list.appendBarrier();
_command_list.appendNpuTimestamp(reinterpret_cast<uint64_t*>(_npu_profiling->npu_ts_infer_start));
}
for (const auto& desc : executor->outputs_desc_map()) {
const std::shared_ptr<ov::ITensor>& outputTensor = tensors.at(desc.first);
size_t outputTensorByteSize = outputTensor->get_byte_size();
executor->setArgumentValue(
desc.second.idx,
static_cast<unsigned char*>(outputTensor->data()) + (i * outputTensorByteSize) / batch_size);
}
_command_list.appendGraphExecute(executor->graph(), profiling_handle);
/// append timestamp command if feature was activated
if (_npu_profiling != nullptr) {
_command_lists.at(i)->appendBarrier();
_command_lists.at(i)->appendNpuTimestamp(
reinterpret_cast<uint64_t*>(_npu_profiling->npu_ts_infer_start));
}
/// append timestamp command if feature was activated
if (_npu_profiling != nullptr) {
_command_list.appendBarrier();
_command_list.appendNpuTimestamp(reinterpret_cast<uint64_t*>(_npu_profiling->npu_ts_infer_end));
}
_command_lists.at(i)->appendGraphExecute(executor->graph(), profiling_handle);
// appendBarrier used in L0 as well
if (!sync_output_with_fences_) {
_command_list.appendBarrier();
_event.AppendSignalEvent(_command_list);
/// append timestamp command if feature was activated
if (_npu_profiling != nullptr) {
_command_lists.at(i)->appendBarrier();
_command_lists.at(i)->appendNpuTimestamp(reinterpret_cast<uint64_t*>(_npu_profiling->npu_ts_infer_end));
}
// appendBarrier used in L0 as well
if (!sync_output_with_fences_) {
_command_lists.at(i)->appendBarrier();
_events.at(i)->AppendSignalEvent(*_command_lists.at(i));
}
_command_lists.at(i)->close();
}
_command_list.close();
}
IntegratedPipeline(const IntegratedPipeline&) = delete;
IntegratedPipeline& operator=(const IntegratedPipeline&) = delete;
virtual ~IntegratedPipeline() = default;
void push() override {
void push(size_t batch_index) override {
OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_IP_PUSH, itt::domains::LevelZeroBackend, "IntegratedPipeline", "push");
if (sync_output_with_fences_) {
_command_queue.executeCommandList(_command_list, _fence);
_command_queue.executeCommandList(*_command_lists.at(batch_index), *_fences.at(batch_index));
} else {
_command_queue.executeCommandList(_command_list);
_command_queue.executeCommandList(*_command_lists.at(batch_index));
}
};
void pull() override {
void pull(size_t batch_index) override {
OV_ITT_TASK_CHAIN(ZERO_EXECUTOR_IP_PULL, itt::domains::LevelZeroBackend, "IntegratedPipeline", "pull");
if (sync_output_with_fences_) {
_fence.hostSynchronize();
_fences.at(batch_index)->hostSynchronize();
} else {
_event.hostSynchronize();
_events.at(batch_index)->hostSynchronize();
}
/// sample npu timestamps if feature was activated
if (_npu_profiling != nullptr) {
@ -220,21 +237,21 @@ public:
}
};
void reset() const override {
void reset(size_t batch_index) const override {
if (sync_output_with_fences_) {
_fence.reset();
_fences.at(batch_index)->reset();
} else {
_event.reset();
_events.at(batch_index)->reset();
}
};
private:
const Config _config;
CommandQueue& _command_queue;
CommandList _command_list;
Fence _fence;
std::vector<std::unique_ptr<CommandList>> _command_lists;
std::vector<std::unique_ptr<Fence>> _fences;
EventPool _event_pool;
Event _event;
std::vector<std::unique_ptr<Event>> _events;
bool sync_output_with_fences_ = true;
std::shared_ptr<zeroProfiling::NpuInferProfiling> _npu_profiling;
};
@ -244,7 +261,8 @@ std::unique_ptr<Pipeline> makePipeline(const std::shared_ptr<const IExecutor>& e
zeroProfiling::ProfilingPool& profiling_pool,
zeroProfiling::ProfilingQuery& profiling_query,
std::shared_ptr<zeroProfiling::NpuInferProfiling> npu_profiling,
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors) {
std::unordered_map<std::string, std::shared_ptr<ov::ITensor>>& tensors,
const size_t batch_size) {
OV_ITT_SCOPED_TASK(itt::domains::LevelZeroBackend, "Infer_request::makePipeline");
if (profiling_pool.create())
profiling_query.create(profiling_pool._handle);
@ -271,7 +289,8 @@ std::unique_ptr<Pipeline> makePipeline(const std::shared_ptr<const IExecutor>& e
npu_profiling,
*command_queues[stage::EXECUTE],
group_ordinal,
tensors);
tensors,
batch_size);
}
return std::make_unique<DiscretePipeline>(config,

79
src/plugins/intel_npu/src/compiler/include/zero_compiler_in_driver.hpp
View File

@ -13,13 +13,10 @@
namespace intel_npu {
namespace driverCompilerAdapter {
#define NotSupportLogHandle(T) \
(std::is_same<T, ze_graph_dditable_ext_t>::value || std::is_same<T, ze_graph_dditable_ext_1_1_t>::value || \
std::is_same<T, ze_graph_dditable_ext_1_2_t>::value || std::is_same<T, ze_graph_dditable_ext_1_3_t>::value)
#define NotSupportLogHandle(T) \
(std::is_same<T, ze_graph_dditable_ext_1_2_t>::value || std::is_same<T, ze_graph_dditable_ext_1_3_t>::value)
#define NotSupportQuery(T) \
(std::is_same<T, ze_graph_dditable_ext_t>::value || std::is_same<T, ze_graph_dditable_ext_1_1_t>::value || \
std::is_same<T, ze_graph_dditable_ext_1_2_t>::value)
#define NotSupportQuery(T) (std::is_same<T, ze_graph_dditable_ext_1_2_t>::value)
// ext version == 1.3 && 1.4, support API (pfnQueryNetworkCreate, pfnQueryNetworkDestroy,
// pfnQueryNetworkGetSupportedLayers)
@ -31,10 +28,14 @@ namespace driverCompilerAdapter {
// For ext version >= 1.5, pfnCreate2 api is avaible
#define NotSupportGraph2(T) \
(std::is_same<T, ze_graph_dditable_ext_t>::value || std::is_same<T, ze_graph_dditable_ext_1_1_t>::value || \
std::is_same<T, ze_graph_dditable_ext_1_2_t>::value || std::is_same<T, ze_graph_dditable_ext_1_3_t>::value || \
(std::is_same<T, ze_graph_dditable_ext_1_2_t>::value || std::is_same<T, ze_graph_dditable_ext_1_3_t>::value || \
std::is_same<T, ze_graph_dditable_ext_1_4_t>::value)
// For ext version >= 1.6, originalShape is avaible
#define NotSupportOriginalShape(T) \
(std::is_same<T, ze_graph_dditable_ext_1_2_t>::value || std::is_same<T, ze_graph_dditable_ext_1_3_t>::value || \
std::is_same<T, ze_graph_dditable_ext_1_4_t>::value || std::is_same<T, ze_graph_dditable_ext_1_5_t>::value)
/**
* Adapter to use CiD through ZeroAPI
*/
@ -71,8 +72,9 @@ public:
* --outputs_precisions="<output1Name>:<output1Precision>"
* --outputs_layouts="<output1Name>:<output1Layout>"
*
* Since the layout information is no longer an important part of the metadata values when using the 2.0 OV API, the
* layout fields shall be filled with default values in order to assure the backward compatibility with the driver.
* Since the layout information is no longer an important part of the metadata values when using the 2.0 OV
* API, the layout fields shall be filled with default values in order to assure the backward compatibility
* with the driver.
*/
static std::string serializeIOInfo(const std::shared_ptr<const ov::Model>& model);
@ -84,21 +86,22 @@ private:
/**
* @brief Extracts the layout value or the state descriptor from the given Level Zero structure.
* @details Extracting the layout information is required only when using older driver versions which rely on this
* legacy attribute. Since this information is not found within the parameter/result nodes, we need to extract this
* value here.
* @details Extracting the layout information is required only when using older driver versions which rely on
* this legacy attribute. Since this information is not found within the parameter/result nodes, we need to
* extract this value here.
*
* The state variables are also not found in the previously mentioned nodes, thus if the given Level Zero parameter
* corresponds to an input/output, we shall extract the layout value from it. Else it represents a state variable
* and the descriptor will be extracted and stored in an OpenVINO specific format.
* The state variables are also not found in the previously mentioned nodes, thus if the given Level Zero
* parameter corresponds to an input/output, we shall extract the layout value from it. Else it represents a
* state variable and the descriptor will be extracted and stored in an OpenVINO specific format.
* @param parameters Holds the already extracted input node descriptors. The transposed shape attribute of the
* corresponding entry may be updated according to the extracted layout value.
* @param results Holds the already extracted output node descriptors. The transposed shape attribute of the
* corresponding entry may be updated according to the extracted layout value.
* @param states The state descriptors shall be stored here in an OpenVINO specific format.
* @param stateNames The output location of the state variables' names in the order found within the compiled model.
* @param arg The Level Zero specific structure from which the layout value or state variable descriptor shall be
* extracted.
* @param stateNames The output location of the state variables' names in the order found within the compiled
* model.
* @param arg The Level Zero specific structure from which the layout value or state variable descriptor shall
* be extracted.
*/
template <typename T>
void getLayoutOrStateDescriptor(IONodeDescriptorMap& parameters,
@ -107,34 +110,18 @@ private:
std::vector<std::string>& stateNames,
const T& arg) const;
/**
* @brief Extracts the node or state descriptor from the given Level Zero structure.
* @details The first version of the "ze_graph_dditable" extension does not directly support extracting the
* information corresponding to the parameter/result nodes. However, the same information can be extracted from the
* legacy I/O structures, with the exception of the node and tensor names fields.
*
* Thus, the current function is a best effort method of assuring the backward compatibility with the driver.
* The missing names have been replaced with the only one available in the structure (i.e. "arg.name"), this
* could potentially lead to issues further in the application's flow.
* @param parameters The parameter node descriptors shall be stored here in an OpenVINO specific format.
* @param results The result node descriptors shall be stored here in an OpenVINO specific format.
* @param states The state descriptors shall be stored here in an OpenVINO specific format.
* @param inputNames The output location of the input names in the order found within the compiled model.
* @param outputNames The output location of the output names in the order found within the compiled model.
* @param stateNames The output location of the state variables' names in the order found within the compiled model.
* @param arg The Level Zero specific structure from which the node or state variable descriptor shall be
* extracted.
*/
template <typename T = TableExtension,
std::enable_if_t<std::is_same<T, ze_graph_dditable_ext_t>::value, bool> = true>
void getNodeOrStateDescriptorLegacy(IONodeDescriptorMap& parameters,
IONodeDescriptorMap& results,
IONodeDescriptorMap& states,
std::vector<std::string>& inputNames,
std::vector<std::string>& outputNames,
std::vector<std::string>& stateNames,
const ze_graph_argument_properties_t& arg) const;
template <typename T = TableExtension, typename std::enable_if_t<NotSupportOriginalShape(T), bool> = true>
void getMetadata(TableExtension* graphDdiTableExt,
ze_graph_handle_t graphHandle,
uint32_t index,
std::vector<std::string>& inputNames,
std::vector<std::string>& outputNames,
std::vector<std::string>& stateNames,
IONodeDescriptorMap& parameters,
IONodeDescriptorMap& results,
IONodeDescriptorMap& state) const;
template <typename T = TableExtension, typename std::enable_if_t<!NotSupportOriginalShape(T), bool> = true>
void getMetadata(TableExtension* graphDdiTableExt,
ze_graph_handle_t graphHandle,
uint32_t index,

31
src/plugins/intel_npu/src/compiler/src/driver_compiler_adapter.cpp
View File

@ -133,7 +133,10 @@ LevelZeroCompilerAdapter::LevelZeroCompilerAdapter() : _logger("LevelZeroCompile
#if defined(NPU_PLUGIN_DEVELOPER_BUILD)
auto adapterManualConfig = std::getenv("ADAPTER_MANUAL_CONFIG");
if (adapterManualConfig != nullptr) {
if (strcmp(adapterManualConfig, "ZE_extension_graph_1_5") == 0) {
if (strcmp(adapterManualConfig, "ZE_extension_graph_1_6") == 0) {
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_6");
targetVersion = ZE_GRAPH_EXT_VERSION_1_6;
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_5") == 0) {
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_5");
targetVersion = ZE_GRAPH_EXT_VERSION_1_5;
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_4") == 0) {
@ -145,26 +148,13 @@ LevelZeroCompilerAdapter::LevelZeroCompilerAdapter() : _logger("LevelZeroCompile
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_2") == 0) {
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_2");
targetVersion = ZE_GRAPH_EXT_VERSION_1_2;
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_1") == 0) {
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_1");
targetVersion = ZE_GRAPH_EXT_VERSION_1_1;
} else if (strcmp(adapterManualConfig, "ZE_extension_graph_1_0") == 0) {
_logger.info("With ADAPTER_MANUAL_CONFIG. Using ZE_GRAPH_EXT_VERSION_1_0");
targetVersion = ZE_GRAPH_EXT_VERSION_1_0;
} else {
OPENVINO_THROW("Using unsupported ADAPTER_MANUAL_CONFIG!");
}
}
#endif
if (ZE_GRAPH_EXT_VERSION_1_1 == targetVersion) {
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_1");
apiAdapter =
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_1_t>>(graphExtName, _driverHandle);
} else if (ZE_GRAPH_EXT_VERSION_1_2 == targetVersion) {
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_2");
apiAdapter =
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_2_t>>(graphExtName, _driverHandle);
} else if (strcmp(graphExtName, ZE_GRAPH_EXT_NAME_1_3) == 0) {
if (ZE_GRAPH_EXT_VERSION_1_3 == targetVersion) {
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_3");
apiAdapter =
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_3_t>>(graphExtName, _driverHandle);
} else if (ZE_GRAPH_EXT_VERSION_1_4 == targetVersion) {
@ -175,9 +165,14 @@ LevelZeroCompilerAdapter::LevelZeroCompilerAdapter() : _logger("LevelZeroCompile
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_5");
apiAdapter =
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_5_t>>(graphExtName, _driverHandle);
} else if (ZE_GRAPH_EXT_VERSION_1_6 == targetVersion) {
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_6");
apiAdapter =
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_6_t>>(graphExtName, _driverHandle);
} else {
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_0");
apiAdapter = std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_t>>(graphExtName, _driverHandle);
_logger.info("Using ZE_GRAPH_EXT_VERSION_1_2");
apiAdapter =
std::make_shared<LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_2_t>>(graphExtName, _driverHandle);
}
}

255
src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp
View File

@ -56,49 +56,6 @@ void checkedMemcpy(void* destination, size_t destinationSize, void const* source
memcpy(destination, source, numberOfBytes);
}
ov::element::Type_t toOVElementType(const ze_graph_metadata_type zeElementType) {
switch (zeElementType) {
case ZE_GRAPH_METADATA_TYPE_UNDEFINED:
return ov::element::Type_t::undefined;
case ZE_GRAPH_METADATA_TYPE_DYNAMIC:
return ov::element::Type_t::dynamic;
case ZE_GRAPH_METADATA_TYPE_BOOLEAN:
return ov::element::Type_t::boolean;
case ZE_GRAPH_METADATA_TYPE_BF16:
return ov::element::Type_t::bf16;
case ZE_GRAPH_METADATA_TYPE_F16:
return ov::element::Type_t::f16;
case ZE_GRAPH_METADATA_TYPE_F32:
return ov::element::Type_t::f32;
case ZE_GRAPH_METADATA_TYPE_F64:
return ov::element::Type_t::f64;
case ZE_GRAPH_METADATA_TYPE_I4:
return ov::element::Type_t::i4;
case ZE_GRAPH_METADATA_TYPE_I8:
return ov::element::Type_t::i8;
case ZE_GRAPH_METADATA_TYPE_I16:
return ov::element::Type_t::i16;
case ZE_GRAPH_METADATA_TYPE_I32:
return ov::element::Type_t::i32;
case ZE_GRAPH_METADATA_TYPE_I64:
return ov::element::Type_t::i64;
case ZE_GRAPH_METADATA_TYPE_U1:
return ov::element::Type_t::u1;
case ZE_GRAPH_METADATA_TYPE_U4:
return ov::element::Type_t::u4;
case ZE_GRAPH_METADATA_TYPE_U8:
return ov::element::Type_t::u8;
case ZE_GRAPH_METADATA_TYPE_U16:
return ov::element::Type_t::u16;
case ZE_GRAPH_METADATA_TYPE_U32:
return ov::element::Type_t::u32;
case ZE_GRAPH_METADATA_TYPE_U64:
return ov::element::Type_t::u64;
default:
return ov::element::Type_t::undefined;
}
}
ov::element::Type_t toOVElementType(const ze_graph_argument_precision_t zeElementType) {
switch (zeElementType) {
case ZE_GRAPH_ARGUMENT_PRECISION_UNKNOWN:
@ -466,7 +423,7 @@ std::string LevelZeroCompilerInDriver<TableExtension>::serializeConfig(
}
/// Stepping and max_tiles are not supported in versions < 5.3 - need to remove it
if ((compilerVersion.major < 5) || (compilerVersion.major <= 5 && compilerVersion.minor < 3)) {
if ((compilerVersion.major < 5) || (compilerVersion.major == 5 && compilerVersion.minor < 3)) {
std::ostringstream stepstr;
stepstr << ov::intel_npu::stepping.name() << KEY_VALUE_SEPARATOR << VALUE_DELIMITER << "\\d+"
<< VALUE_DELIMITER;
@ -478,8 +435,9 @@ std::string LevelZeroCompilerInDriver<TableExtension>::serializeConfig(
_logger.warning("NPU_MAX_TILES property is not suppored by this compiler version. Removing from parameters");
content = std::regex_replace(content, std::regex(maxtilestr.str()), "");
}
/// Removing INFERENCE_PRECISION_HINT for older compilers
if ((compilerVersion.major < 5) || (compilerVersion.major <= 5 && compilerVersion.minor < 4)) {
if ((compilerVersion.major < 5) || (compilerVersion.major == 5 && compilerVersion.minor < 4)) {
std::ostringstream precstr;
precstr << ov::hint::inference_precision.name() << KEY_VALUE_SEPARATOR << VALUE_DELIMITER << "\\S+"
<< VALUE_DELIMITER;
@ -487,6 +445,7 @@ std::string LevelZeroCompilerInDriver<TableExtension>::serializeConfig(
"INFERENCE_PRECISION_HINT property is not suppored by this compiler version. Removing from parameters");
content = std::regex_replace(content, std::regex(precstr.str()), "");
}
/// Replacing NPU_TILES (for all versions) with NPU_DPU_GROUPS for backwards compatibility
if (std::regex_search(content, std::regex(ov::intel_npu::tiles.name()))) {
_logger.warning("NPU_TILES property is not suppored by this compiler version. Swaping it to "
@ -494,6 +453,15 @@ std::string LevelZeroCompilerInDriver<TableExtension>::serializeConfig(
content = std::regex_replace(content, std::regex(ov::intel_npu::tiles.name()), "NPU_DPU_GROUPS");
}
// Batch mode property is not supported in versions < 5.5 - need to remove it
if ((compilerVersion.major < 5) || (compilerVersion.major == 5 && compilerVersion.minor < 5)) {
std::ostringstream batchstr;
batchstr << ov::intel_npu::batch_mode.name() << KEY_VALUE_SEPARATOR << VALUE_DELIMITER << "\\S+"
<< VALUE_DELIMITER;
_logger.warning("NPU_BATCH_MODE property is not suppored by this compiler version. Removing from parameters");
content = std::regex_replace(content, std::regex(batchstr.str()), "");
}
return "--config " + content;
}
@ -910,50 +878,6 @@ void LevelZeroCompilerInDriver<TableExtension>::getLayoutOrStateDescriptor(IONod
}
}
template <typename TableExtension>
template <typename T, std::enable_if_t<std::is_same<T, ze_graph_dditable_ext_t>::value, bool>>
void LevelZeroCompilerInDriver<TableExtension>::getNodeOrStateDescriptorLegacy(
IONodeDescriptorMap& parameters,
IONodeDescriptorMap& results,
IONodeDescriptorMap& states,
std::vector<std::string>& inputNames,
std::vector<std::string>& outputNames,
std::vector<std::string>& stateNames,
const ze_graph_argument_properties_t& arg) const {
std::string legacyName = arg.name;
const ov::element::Type_t precision = toOVElementType(arg.devicePrecision);
// The layout shall differ from the default one only when using significantly older drivers. In order to accommodate
// this case, an extra attribute needs to be stored which holds the transposed shape.
const std::vector<size_t> originalDimensions(arg.dims, arg.dims + zeLayoutToRank(arg.deviceLayout));
const std::vector<size_t> reshapedDimensions = reshapeByLayout(originalDimensions, arg.deviceLayout);
const ov::Shape originalShape = ov::Shape(originalDimensions);
const ov::Shape transposedShape = ov::Shape(reshapedDimensions);
if (!isStateInputName(legacyName) && !isStateOutputName(legacyName)) {
if (arg.type == ZE_GRAPH_ARGUMENT_TYPE_INPUT) {
_logger.info("getNodeOrStateDescriptorLegacy Found input \"%s\"", legacyName.c_str());
inputNames.push_back(legacyName);
parameters[legacyName] = {legacyName, legacyName, {legacyName}, precision, originalShape, transposedShape};
}
if (arg.type == ZE_GRAPH_ARGUMENT_TYPE_OUTPUT) {
_logger.info("getNodeOrStateDescriptorLegacy Found output \"%s\"", legacyName.c_str());
outputNames.push_back(legacyName);
results[legacyName] = {legacyName, legacyName, {legacyName}, precision, originalShape, transposedShape};
}
} else if (isStateInputName(legacyName)) {
// The inputs and outputs of the state nodes share the same metadata, thus we'll consider only the the inputs
// here
legacyName = legacyName.substr(READVALUE_PREFIX.length());
_logger.info("getNodeOrStateDescriptorLegacy Found state variable \"%s\"", legacyName.c_str());
stateNames.push_back(legacyName);
states[legacyName] = {legacyName, "", {}, precision, originalShape, originalShape};
}
}
/**
* @brief Extracts the parameter/result (i.e. input/output) descriptors from Level Zero specific structures into
* OpenVINO specific ones.
@ -961,26 +885,6 @@ void LevelZeroCompilerInDriver<TableExtension>::getNodeOrStateDescriptorLegacy(
* @param names The I/O identifiers shall be stored here in the order found within the compiled model.
* @param metadata The Level Zero structure fomr which the descriptors will be extracted.
*/
static void getNodeDescriptor(IONodeDescriptorMap& nodeDescriptors,
std::vector<std::string>& names,
ze_graph_argument_metadata_t& metadata) {
const ov::element::Type_t precision = toOVElementType(metadata.data_type);
ov::Shape shape;
std::unordered_set<std::string> outputTensorNames;
for (uint32_t id = 0; id < metadata.tensor_names_count; id++) {
outputTensorNames.insert(metadata.tensor_names[id]);
}
for (uint32_t id = 0; id < metadata.shape_size; id++) {
shape.push_back(metadata.shape[id]);
}
const std::string& legacyName = metadata.input_name;
names.push_back(legacyName);
nodeDescriptors[legacyName] =
{legacyName, metadata.friendly_name, std::move(outputTensorNames), precision, shape, shape};
}
static void getNodeDescriptor(IONodeDescriptorMap& nodeDescriptors,
std::vector<std::string>& names,
ze_graph_argument_properties_3_t& arg) {
@ -991,9 +895,11 @@ static void getNodeDescriptor(IONodeDescriptorMap& nodeDescriptors,
for (uint32_t id = 0; id < arg.associated_tensor_names_count; id++) {
outputTensorNames.insert(arg.associated_tensor_names[id]);
}
for (uint32_t id = 0; id < arg.dims_count; id++) {
shape.push_back(arg.dims[id]);
}
const std::string& legacyName = arg.name;
names.push_back(arg.debug_friendly_name);
@ -1001,77 +907,35 @@ static void getNodeDescriptor(IONodeDescriptorMap& nodeDescriptors,
{legacyName, arg.debug_friendly_name, std::move(outputTensorNames), precision, shape, shape};
}
template <>
void LevelZeroCompilerInDriver<ze_graph_dditable_ext_t>::getMetadata(ze_graph_dditable_ext_t* graphDdiTableExt,
ze_graph_handle_t graphHandle,
uint32_t index,
std::vector<std::string>& inputNames,
std::vector<std::string>& outputNames,
std::vector<std::string>& stateNames,
IONodeDescriptorMap& parameters,
IONodeDescriptorMap& results,
IONodeDescriptorMap& states) const {
ze_graph_argument_properties_t arg;
auto result = graphDdiTableExt->pfnGetArgumentProperties(graphHandle, index, &arg);
if (ZE_RESULT_SUCCESS != result) {
OPENVINO_THROW("L0 pfnGetArgumentProperties",
" result: ",
ze_result_to_string(result),
", code 0x",
std::hex,
uint64_t(result));
static void getNodeDescriptor(IONodeDescriptorMap& nodeDescriptors,
std::vector<std::string>& names,
ze_graph_argument_properties_3_t& arg,
ze_graph_argument_metadata_t& metadata) {
ov::element::Type_t precision = toOVElementType(arg.devicePrecision);
ov::Shape transposedShape, originalShape;
std::unordered_set<std::string> outputTensorNames;
for (uint32_t id = 0; id < arg.associated_tensor_names_count; id++) {
outputTensorNames.insert(arg.associated_tensor_names[id]);
}
getNodeOrStateDescriptorLegacy(parameters, results, states, inputNames, outputNames, stateNames, arg);
}
template <>
void LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_1_t>::getMetadata(ze_graph_dditable_ext_1_1_t* graphDdiTableExt,
ze_graph_handle_t graphHandle,
uint32_t index,
std::vector<std::string>& inputNames,
std::vector<std::string>& outputNames,
std::vector<std::string>& stateNames,
IONodeDescriptorMap& parameters,
IONodeDescriptorMap& results,
IONodeDescriptorMap& states) const {
ze_graph_argument_properties_2_t arg;
auto result = graphDdiTableExt->pfnGetArgumentProperties2(graphHandle, index, &arg);
if (ZE_RESULT_SUCCESS != result) {
OPENVINO_THROW("L0 pfnGetArgumentProperties2",
" result: ",
ze_result_to_string(result),
", code 0x",
std::hex,
uint64_t(result));
for (uint32_t id = 0; id < arg.dims_count; id++) {
transposedShape.push_back(arg.dims[id]);
}
// The I/O data corresponding to the states of the model is not found within the OpenVINO 2.0 attributes contained
// by the compiled model, thus we should not query them
if (!isStateInputName(arg.name) && !isStateOutputName(arg.name)) {
ze_graph_argument_metadata_t metadata;
result = graphDdiTableExt->pfnGraphGetArgumentMetadata(graphHandle, index, &metadata);
if (ZE_RESULT_SUCCESS != result) {
OPENVINO_THROW("L0 pfnGraphGetArgumentMetadata",
" result: ",
ze_result_to_string(result),
", code 0x",
std::hex,
uint64_t(result));
}
if (ZE_GRAPH_ARGUMENT_TYPE_INPUT == arg.type) {
getNodeDescriptor(parameters, inputNames, metadata);
}
if (ZE_GRAPH_ARGUMENT_TYPE_OUTPUT == arg.type) {
getNodeDescriptor(results, outputNames, metadata);
}
for (uint32_t id = 0; id < metadata.shape_size; id++) {
originalShape.push_back(metadata.shape[id]);
}
getLayoutOrStateDescriptor(parameters, results, states, stateNames, arg);
const std::string& legacyName = arg.name;
names.push_back(arg.debug_friendly_name);
nodeDescriptors[arg.debug_friendly_name] =
{legacyName, arg.debug_friendly_name, std::move(outputTensorNames), precision, originalShape, transposedShape};
}
template <typename TableExtension>
template <typename T, std::enable_if_t<NotSupportOriginalShape(T), bool>>
void LevelZeroCompilerInDriver<TableExtension>::getMetadata(TableExtension* graphDdiTableExt,
ze_graph_handle_t graphHandle,
uint32_t index,
@ -1105,6 +969,52 @@ void LevelZeroCompilerInDriver<TableExtension>::getMetadata(TableExtension* grap
getLayoutOrStateDescriptor(parameters, results, states, stateNames, arg);
}
template <typename TableExtension>
template <typename T, std::enable_if_t<!NotSupportOriginalShape(T), bool>>
void LevelZeroCompilerInDriver<TableExtension>::getMetadata(TableExtension* graphDdiTableExt,
ze_graph_handle_t graphHandle,
uint32_t index,
std::vector<std::string>& inputNames,
std::vector<std::string>& outputNames,
std::vector<std::string>& stateNames,
IONodeDescriptorMap& parameters,
IONodeDescriptorMap& results,
IONodeDescriptorMap& states) const {
ze_graph_argument_properties_3_t arg;
auto result = graphDdiTableExt->pfnGetArgumentProperties3(graphHandle, index, &arg);
if (ZE_RESULT_SUCCESS != result) {
OPENVINO_THROW("L0 pfnGetArgumentProperties3",
" result: ",
ze_result_to_string(result),
", code 0x",
std::hex,
uint64_t(result));
}
if (!isStateInputName(arg.name) && !isStateOutputName(arg.name)) {
ze_graph_argument_metadata_t metadata;
result = graphDdiTableExt->pfnGraphGetArgumentMetadata(graphHandle, index, &metadata);
if (ZE_RESULT_SUCCESS != result) {
OPENVINO_THROW("L0 pfnGraphGetArgumentMetadata",
" result: ",
ze_result_to_string(result),
", code 0x",
std::hex,
uint64_t(result));
}
if (ZE_GRAPH_ARGUMENT_TYPE_INPUT == arg.type) {
getNodeDescriptor(parameters, inputNames, arg, metadata);
}
if (ZE_GRAPH_ARGUMENT_TYPE_OUTPUT == arg.type) {
getNodeDescriptor(results, outputNames, arg, metadata);
}
}
getLayoutOrStateDescriptor(parameters, results, states, stateNames, arg);
}
template <typename TableExtension>
NetworkMetadata LevelZeroCompilerInDriver<TableExtension>::getNetworkMeta(ze_graph_handle_t graphHandle) const {
ze_graph_properties_t graphProperties{};
@ -1173,12 +1083,11 @@ std::string LevelZeroCompilerInDriver<TableExtension>::getLatestBuildError() con
return logContent;
}
template class LevelZeroCompilerInDriver<ze_graph_dditable_ext_t>;
template class LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_1_t>;
template class LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_2_t>;
template class LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_3_t>;
template class LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_4_t>;
template class LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_5_t>;
template class LevelZeroCompilerInDriver<ze_graph_dditable_ext_1_6_t>;
} // namespace driverCompilerAdapter
} // namespace intel_npu

1
src/plugins/intel_npu/src/plugin/include/backends.hpp
View File

@ -31,6 +31,7 @@ public:
std::string getBackendName() const;
uint32_t getDriverVersion() const;
uint32_t getDriverExtVersion() const;
bool isBatchingSupported() const;
void registerOptions(OptionsDesc& options) const;
std::string getCompilationPlatform(const std::string_view platform, const std::string& deviceId) const;

8
src/plugins/intel_npu/src/plugin/src/backends.cpp
View File

@ -155,6 +155,14 @@ uint32_t NPUBackends::getDriverExtVersion() const {
OPENVINO_THROW("No available backend");
}
bool NPUBackends::isBatchingSupported() const {
if (_backend != nullptr) {
return _backend->isBatchingSupported();
}
OPENVINO_THROW("No available backend");
}
std::shared_ptr<IDevice> NPUBackends::getDevice(const std::string& specificName) const {
_logger.debug("Searching for device %s to use started...", specificName.c_str());
// TODO iterate over all available backends

6
src/plugins/intel_npu/src/plugin/src/compiled_model.cpp
View File

@ -308,6 +308,12 @@ void CompiledModel::initialize_properties() {
[](const Config& config) {
return config.get<CREATE_EXECUTOR>();
}}},
{ov::intel_npu::batch_mode.name(),
{false,
ov::PropertyMutability::RO,
[](const Config& config) {
return config.getString<BATCH_MODE>();
}}},
};
for (auto& property : _properties) {

86
src/plugins/intel_npu/src/plugin/src/plugin.cpp
View File

@ -37,18 +37,28 @@ const char* NPU_PLUGIN_LIB_NAME = "openvino_intel_npu_plugin";
* @param resultDescriptors Describes the output nodes.
* @param inputNames The names of the inputs registered in the order given by the model.
* @param outputNames The names of the outputs registered in the order given by the model.
* @param isBatchingSupported Newer driver versions support batching mode on the plugin.
*/
std::shared_ptr<ov::Model> create_dummy_model(const IONodeDescriptorMap& parameterDescriptors,
const IONodeDescriptorMap& resultDescriptors,
const std::vector<std::string>& inputNames,
const std::vector<std::string>& outputNames) {
const std::vector<std::string>& outputNames,
bool isBatchingSupported) {
ov::ParameterVector parameters;
ov::NodeVector results;
for (const std::string& inputName : inputNames) {
const IONodeDescriptor& parameterDescriptor = parameterDescriptors.at(inputName);
std::shared_ptr<ov::op::v0::Parameter> parameter =
std::make_shared<ov::op::v0::Parameter>(parameterDescriptor.precision, parameterDescriptor.transposedShape);
std::shared_ptr<ov::op::v0::Parameter> parameter = [&] {
if (isBatchingSupported) {
return std::make_shared<ov::op::v0::Parameter>(parameterDescriptor.precision,
parameterDescriptor.originalShape);
}
return std::make_shared<ov::op::v0::Parameter>(parameterDescriptor.precision,
parameterDescriptor.transposedShape);
}();
parameter->set_friendly_name(parameterDescriptor.currentNodeName);
parameter->output(0).get_tensor().set_names(parameterDescriptor.outputTensorNames);
parameters.push_back(parameter);
@ -65,10 +75,16 @@ std::shared_ptr<ov::Model> create_dummy_model(const IONodeDescriptorMap& paramet
std::make_shared<ov::op::v0::Constant>(resultDescriptor.precision, CONSTANT_NODE_DUMMY_SHAPE);
constantDummy->set_friendly_name(resultDescriptor.legacyName);
const std::shared_ptr<ov::descriptor::Tensor>& tensorDummy =
std::make_shared<ov::descriptor::Tensor>(resultDescriptor.precision,
resultDescriptor.transposedShape,
resultDescriptor.outputTensorNames);
const std::shared_ptr<ov::descriptor::Tensor>& tensorDummy = [&] {
if (isBatchingSupported) {
return std::make_shared<ov::descriptor::Tensor>(resultDescriptor.precision,
resultDescriptor.originalShape,
resultDescriptor.outputTensorNames);
}
return std::make_shared<ov::descriptor::Tensor>(resultDescriptor.precision,
resultDescriptor.transposedShape,
resultDescriptor.outputTensorNames);
}();
std::shared_ptr<ov::Node> result = std::make_shared<ov::op::v0::Result>(constantDummy);
result->output(0).set_tensor_ptr(tensorDummy);
@ -79,6 +95,33 @@ std::shared_ptr<ov::Model> create_dummy_model(const IONodeDescriptorMap& paramet
return std::make_shared<ov::Model>(results, parameters);
}
/**
* @brief Setting batching mode
* @details In the case of older drivers or discrete platforms, we force batching to compiler mode since it is not
* supported. Othwersie set it tu AUTO if this wasn't set by the user
* @param isBatchingSupported Newer driver versions support batching mode on the plugin.
* @param config A configuration map.
*/
void set_batch_config(bool isBatchingSupported, Config& config) {
if (!isBatchingSupported || config.get<PLATFORM>() == ov::intel_npu::Platform::NPU3700) {
if (config.has<BATCH_MODE>()) {
if (config.get<BATCH_MODE>() == ov::intel_npu::BatchMode::PLUGIN) {
OPENVINO_THROW("Batching on plugin is not supported with this driver version");
}
}
std::stringstream strStream;
strStream << ov::intel_npu::BatchMode::COMPILER;
config.update({{ov::intel_npu::batch_mode.name(), strStream.str()}});
}
if (!config.has<BATCH_MODE>()) {
std::stringstream strStream;
strStream << ov::intel_npu::BatchMode::AUTO;
config.update({{ov::intel_npu::batch_mode.name(), strStream.str()}});
}
}
std::map<std::string, std::string> any_copy(const ov::AnyMap& params) {
std::map<std::string, std::string> result;
for (auto&& value : params) {
@ -423,6 +466,12 @@ Plugin::Plugin()
[](const Config& config) {
return config.getString<BACKEND_COMPILATION_PARAMS>();
}}},
{ov::intel_npu::batch_mode.name(),
{false,
ov::PropertyMutability::RW,
[](const Config& config) {
return config.getString<BATCH_MODE>();
}}},
};
for (auto& property : _properties) {
@ -485,6 +534,20 @@ std::shared_ptr<ov::ICompiledModel> Plugin::compile_model(const std::shared_ptr<
auto device = _backends->getDevice(localConfig.get<DEVICE_ID>());
localConfig.update({{ov::intel_npu::platform.name(), platform}});
set_batch_config(_backends->isBatchingSupported(), localConfig);
if (!model->get_variables().empty()) {
if (localConfig.get<BATCH_MODE>() == ov::intel_npu::BatchMode::PLUGIN) {
OPENVINO_THROW("This model contains states, thus it is not supported when handling batching on the plugin");
}
_logger.info("The batching will be handled by the compiler due to states found inside the IR");
std::stringstream strStream;
strStream << ov::intel_npu::BatchMode::COMPILER;
localConfig.update({{ov::intel_npu::batch_mode.name(), strStream.str()}});
}
// Update stepping w/ information from driver, unless provided by user or we are off-device
// Ignore, if compilation was requested for platform, different from current
if (!localConfig.has<STEPPING>() && device != nullptr && device->getName() == platform) {
@ -566,6 +629,8 @@ std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& stream, c
localConfig.update({{ov::intel_npu::platform.name(), platform}});
auto device = _backends->getDevice(localConfig.get<DEVICE_ID>());
set_batch_config(_backends->isBatchingSupported(), localConfig);
Logger logger("NPUPlugin", localConfig.get<LOG_LEVEL>());
const auto loadedFromCache = localConfig.get<LOADED_FROM_CACHE>();
@ -590,8 +655,11 @@ std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& stream, c
auto meta = compiler->parse(blob, localConfig);
meta.name = "net" + std::to_string(_compiledModelLoadCounter++);
const std::shared_ptr<ov::Model> modelDummy =
create_dummy_model(meta.parameters, meta.results, meta.inputNames, meta.outputNames);
const std::shared_ptr<ov::Model> modelDummy = create_dummy_model(meta.parameters,
meta.results,
meta.inputNames,
meta.outputNames,
_backends->isBatchingSupported());
bool profiling = localConfig.get<PERF_COUNT>();

2
src/plugins/intel_npu/thirdparty/level-zero-ext vendored

@ -1 +1 @@
Subproject commit 0e1c471356a724ef6d176ba027a68e210d90939e
Subproject commit d490a130fbb80e600b3aed3886c305abcb60d77c