From c0853f0f2f99cbca73d8fad13c0ab29fd2ddde14 Mon Sep 17 00:00:00 2001 From: Mircea-Aurelian Dan Date: Thu, 23 May 2024 14:28:53 +0300 Subject: [PATCH] Upstream NPU behavior tests (#24552) ### Details: - By this PR we want to create `ov_npu_func_tests` executable having functional tests from `NPU plugin` (SLTs and Subgraphs to be added later). - Functional tests were copy-pasted to EISW-120273/original/copy_npu_func_tests_to_ov branch to review changes easier at: https://github.com/MirceaDan99/openvino/compare/d4f0300e1bb2f5e451d8eee2aa1193ff14460451..7b7e177f113544ffcec65f0fa5d0fef60056a274 - Latest validation is: https://vpux-ci-vpuip.ir.intel.com/job/ov_integration/job/ov_integration/1167/ ### Tickets: - *E-120273* --- src/plugins/intel_npu/CMakeLists.txt | 4 + src/plugins/intel_npu/tests/CMakeLists.txt | 10 + .../intel_npu/tests/functional/CMakeLists.txt | 51 + .../intel_npu/tests/functional/README.md | 22 + .../fail_gracefully_forward_compatibility.cpp | 25 + .../fail_gracefully_forward_compatibility.hpp | 137 +++ .../functional/behavior/infer_request_run.cpp | 29 + .../functional/behavior/infer_request_run.hpp | 399 +++++++ .../downgrade_interpolate11.cpp | 19 + .../downgrade_interpolate11.hpp | 97 ++ .../expected_throw.cpp | 16 + .../expected_throw.hpp | 94 ++ .../inputs_outputs.cpp | 19 + .../inputs_outputs.hpp | 121 +++ .../query_network.cpp | 170 +++ .../functional/behavior/replace_sw_layers.cpp | 57 + .../functional/behavior/work_with_devices.cpp | 20 + .../functional/behavior/work_with_devices.hpp | 158 +++ .../tests/functional/common/functions.cpp | 37 + .../tests/functional/common/functions.h | 16 + .../functional/common/npu_test_env_cfg.cpp | 208 ++++ .../functional/common/npu_test_env_cfg.hpp | 83 ++ .../common/print_test_case_name.hpp | 37 + .../functional/common/random_generator.cpp | 14 + .../functional/common/random_generator.hpp | 22 + .../tests/functional/common/tensor_view.hpp | 128 +++ .../tests/functional/common/utils.cpp | 107 ++ .../tests/functional/common/utils.hpp | 91 ++ .../tests/functional/core_config.cpp | 14 + .../internal/overload/compile_and_infer.hpp | 84 ++ .../overload/compiled_model/property.cpp | 333 ++++++ .../overload/ov_infer_request/callback.hpp | 126 +++ .../ov_infer_request/cancellation.hpp | 67 ++ .../overload/ov_infer_request/io_tensor.hpp | 257 +++++ .../ov_infer_request/multithreading.hpp | 102 ++ .../overload/ov_plugin/core_integration.hpp | 223 ++++ .../internal/overload/ov_plugin/life_time.hpp | 226 ++++ .../overload/overload_test_utils_npu.hpp | 35 + .../intel_npu/tests/functional/main.cpp | 85 ++ .../compiled_model/compiled_model_base.cpp | 69 ++ .../behavior/compiled_model/import_export.cpp | 89 ++ .../behavior/compiled_model/properties.cpp | 275 +++++ .../compiled_model/properties_hetero.cpp | 30 + .../ov_infer_request/batched_tensors.cpp | 17 + .../behavior/ov_infer_request/callback.cpp | 56 + .../ov_infer_request/cancellation.cpp | 26 + .../ov_infer_request/compile_and_infer.cpp | 22 + .../ov_infer_request/infer_consistency.cpp | 97 ++ .../behavior/ov_infer_request/inference.cpp | 18 + .../ov_infer_request/inference_chaining.cpp | 26 + .../behavior/ov_infer_request/io_tensor.cpp | 102 ++ .../ov_infer_request/iteration_chaining.cpp | 32 + .../ov_infer_request/memory_states.cpp | 32 + .../ov_infer_request/multithreading.cpp | 56 + .../ov_infer_request/perf_counters.cpp | 57 + .../ov_infer_request/properties_tests.cpp | 20 + .../behavior/ov_infer_request/wait.cpp | 40 + .../behavior/ov_plugin/caching_tests.cpp | 127 +++ .../behavior/ov_plugin/core_integration.cpp | 304 ++++++ .../ov_plugin/core_threading_tests.cpp | 46 + .../behavior/ov_plugin/life_time.cpp | 48 + .../behavior/ov_plugin/properties_tests.cpp | 247 +++++ .../behavior/ov_plugin/query_model.cpp | 23 + .../behavior/ov_plugin/remote.cpp | 27 + .../behavior/ov_plugin/version.cpp | 21 + .../npu_test_report.cpp | 75 ++ .../npu_test_report.hpp | 50 + .../shared_tests_instances/npu_test_tool.cpp | 26 + .../shared_tests_instances/npu_test_tool.hpp | 29 + .../skip_tests_config.cpp | 972 ++++++++++++++++++ 70 files changed, 6802 insertions(+) create mode 100644 src/plugins/intel_npu/tests/CMakeLists.txt create mode 100644 src/plugins/intel_npu/tests/functional/CMakeLists.txt create mode 100644 src/plugins/intel_npu/tests/functional/README.md create mode 100644 src/plugins/intel_npu/tests/functional/behavior/fail_gracefully_forward_compatibility.cpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/fail_gracefully_forward_compatibility.hpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/infer_request_run.cpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/infer_request_run.hpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/downgrade_interpolate11.cpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/downgrade_interpolate11.hpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/expected_throw.cpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/expected_throw.hpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/inputs_outputs.cpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/inputs_outputs.hpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/query_network.cpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/replace_sw_layers.cpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/work_with_devices.cpp create mode 100644 src/plugins/intel_npu/tests/functional/behavior/work_with_devices.hpp create mode 100644 src/plugins/intel_npu/tests/functional/common/functions.cpp create mode 100644 src/plugins/intel_npu/tests/functional/common/functions.h create mode 100644 src/plugins/intel_npu/tests/functional/common/npu_test_env_cfg.cpp create mode 100644 src/plugins/intel_npu/tests/functional/common/npu_test_env_cfg.hpp create mode 100644 src/plugins/intel_npu/tests/functional/common/print_test_case_name.hpp create mode 100644 src/plugins/intel_npu/tests/functional/common/random_generator.cpp create mode 100644 src/plugins/intel_npu/tests/functional/common/random_generator.hpp create mode 100644 src/plugins/intel_npu/tests/functional/common/tensor_view.hpp create mode 100644 src/plugins/intel_npu/tests/functional/common/utils.cpp create mode 100644 src/plugins/intel_npu/tests/functional/common/utils.hpp create mode 100644 src/plugins/intel_npu/tests/functional/core_config.cpp create mode 100644 src/plugins/intel_npu/tests/functional/internal/overload/compile_and_infer.hpp create mode 100644 src/plugins/intel_npu/tests/functional/internal/overload/compiled_model/property.cpp create mode 100644 src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/callback.hpp create mode 100644 src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/cancellation.hpp create mode 100644 src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/io_tensor.hpp create mode 100644 src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/multithreading.hpp create mode 100644 src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/core_integration.hpp create mode 100644 src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/life_time.hpp create mode 100644 src/plugins/intel_npu/tests/functional/internal/overload/overload_test_utils_npu.hpp create mode 100644 src/plugins/intel_npu/tests/functional/main.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/compiled_model_base.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/import_export.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties_hetero.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/batched_tensors.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/callback.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/cancellation.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/compile_and_infer.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/infer_consistency.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/inference.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/inference_chaining.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/io_tensor.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/iteration_chaining.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/memory_states.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/multithreading.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/perf_counters.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/properties_tests.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/wait.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/caching_tests.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_threading_tests.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/life_time.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/query_model.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/remote.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/version.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_report.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_report.hpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_tool.cpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_tool.hpp create mode 100644 src/plugins/intel_npu/tests/functional/shared_tests_instances/skip_tests_config.cpp diff --git a/src/plugins/intel_npu/CMakeLists.txt b/src/plugins/intel_npu/CMakeLists.txt index 5e164845b00..dd9ca927934 100644 --- a/src/plugins/intel_npu/CMakeLists.txt +++ b/src/plugins/intel_npu/CMakeLists.txt @@ -27,3 +27,7 @@ endif() add_subdirectory(thirdparty EXCLUDE_FROM_ALL) add_subdirectory(src) + +if(ENABLE_TESTS) + add_subdirectory(tests) +endif() diff --git a/src/plugins/intel_npu/tests/CMakeLists.txt b/src/plugins/intel_npu/tests/CMakeLists.txt new file mode 100644 index 00000000000..4c41f008eb7 --- /dev/null +++ b/src/plugins/intel_npu/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +if (MSVC) + # NPU plugin tests don't have macros that have `defined` + # in their expansion + ov_add_compiler_flags(/wd5105) +endif() +add_subdirectory(functional) diff --git a/src/plugins/intel_npu/tests/functional/CMakeLists.txt b/src/plugins/intel_npu/tests/functional/CMakeLists.txt new file mode 100644 index 00000000000..297181a67ff --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/CMakeLists.txt @@ -0,0 +1,51 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +if(ENABLE_LTO) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE ON) +endif() + +set(TARGET_NAME ov_npu_func_tests) +set(EXCLUDED_FUNC_TESTS_DIR "") +set(OPTIONAL_FUNC_TESTS_INCLUDES "") +set(OPTIONAL_FUNC_TESTS_LIBS "") + +ov_add_test_target( + NAME ${TARGET_NAME} + ROOT ${CMAKE_CURRENT_SOURCE_DIR} + EXCLUDED_SOURCE_PATHS + ${EXCLUDED_FUNC_TESTS_DIR} + INCLUDES + ${CMAKE_CURRENT_SOURCE_DIR} + ${OPTIONAL_FUNC_TESTS_INCLUDES} + "${CMAKE_CURRENT_SOURCE_DIR}/shared_tests_instances" + "${CMAKE_CURRENT_SOURCE_DIR}/behavior" + "${CMAKE_CURRENT_SOURCE_DIR}/internal" + LINK_LIBRARIES + ${OPTIONAL_FUNC_TESTS_LIBS} + openvino::func_test_utils + openvino::funcSharedTests + openvino::format_reader + openvino::reference + openvino::runtime + openvino::npu_al +) + +set_target_properties(${TARGET_NAME} PROPERTIES + FOLDER ${CMAKE_CURRENT_SOURCE_DIR} + CXX_STANDARD 17) + +if(MSVC) + # Enforce standards conformance on MSVC + target_compile_options(${TARGET_NAME} + PRIVATE + /Zc:preprocessor + ) +endif() + +install(TARGETS ${TARGET_NAME} + RUNTIME DESTINATION tests + COMPONENT tests + EXCLUDE_FROM_ALL +) diff --git a/src/plugins/intel_npu/tests/functional/README.md b/src/plugins/intel_npu/tests/functional/README.md new file mode 100644 index 00000000000..a749093fee4 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/README.md @@ -0,0 +1,22 @@ +# OpenVINO Functional test suite + +Test binary `ov_npu_func_tests` is built with GTest framework and includes test instances which are common +for other OpenVINO plugins as well as test instances specific for this plugin. +`ov_npu_func_tests --help` will show the usage. + +## Environment variables + +The following environment variables can be set up for the run of test binary `ov_npu_func_tests`. Values for boolean variables are specified as `1` or `0`. +* `IE_NPU_TESTS_DUMP_PATH` - string type, target directory for the exported artifacts and source directory for imported artifacts +* `IE_NPU_TESTS_DEVICE_NAME` - string type, passed to the Inference Engine as the name of device for loading the appropriate plugin. Default - `NPU` +* `IE_NPU_TESTS_LOG_LEVEL` - string type, passed to the plugin and allows additional debug output to the console. Sample value - `LOG_DEBUG` +* `IE_NPU_TESTS_RUN_EXPORT` - bool type, denotes whether to export produced networks as artifacts to the files matching current test case +* `IE_NPU_TESTS_RUN_IMPORT` - bool type, denotes whether to import the networks from the files matching current test case (instead of compiling them on the fly) +* `IE_NPU_TESTS_RUN_INFER` - bool type, denotes whether to execute infer request which as part of the test case or not +* `IE_NPU_TESTS_EXPORT_INPUT` - bool type, denotes whether to export produced input data as artifacts to the files matching current test case +* `IE_NPU_TESTS_EXPORT_REF` - bool type, denotes whether to export calculated reference values as artifacts to the files matching current test case +* `IE_NPU_TESTS_IMPORT_INPUT` - bool type, denotes whether to read input data from the files matching current test case (instead of using generated data) +* `IE_NPU_TESTS_IMPORT_REF` - bool type, denotes whether to read reference values from the files matching current test case (instead of calculating them) +* `IE_NPU_TESTS_RAW_EXPORT` - bool type, denotes whether to use header for exported network file or not +* `IE_NPU_TESTS_LONG_FILE_NAME` - bool type, denotes whether to allow longer file names for the exported artifacts. By default shorter file names are used for all operating systems +* `IE_NPU_TESTS_PLATFORM` - string type, enable compiler config option `NPU_PLATFORM` with value from the environment. Sample value - `NPU3700`. Please refer to the documentation for more information about possible values. diff --git a/src/plugins/intel_npu/tests/functional/behavior/fail_gracefully_forward_compatibility.cpp b/src/plugins/intel_npu/tests/functional/behavior/fail_gracefully_forward_compatibility.cpp new file mode 100644 index 00000000000..95405abc0f2 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/fail_gracefully_forward_compatibility.cpp @@ -0,0 +1,25 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/fail_gracefully_forward_compatibility.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; + +bool UnsupportedTestOperation::visit_attributes(AttributeVisitor& /*visitor*/) { + return true; +} + +namespace { + +const std::vector configs = {{}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTest, FailGracefullyTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + FailGracefullyTest::getTestCaseName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/behavior/fail_gracefully_forward_compatibility.hpp b/src/plugins/intel_npu/tests/functional/behavior/fail_gracefully_forward_compatibility.hpp new file mode 100644 index 00000000000..5b56a5788b6 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/fail_gracefully_forward_compatibility.hpp @@ -0,0 +1,137 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "base/ov_behavior_test_utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "common_test_utils/node_builders/constant.hpp" +#include "functional_test_utils/ov_plugin_cache.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "intel_npu/al/config/common.hpp" + +using CompilationParams = std::tuple; + +using ::testing::AllOf; +using ::testing::HasSubstr; + +namespace ov { +namespace test { +namespace behavior { + +class UnsupportedTestOperation : public ov::op::Op { +public: + OPENVINO_OP("UnsupportedTestOperation"); + + UnsupportedTestOperation() = default; + explicit UnsupportedTestOperation(const ov::Output& arg): Op({arg}) { + constructor_validate_and_infer_types(); + } + + void validate_and_infer_types() override { + auto input_pshape = get_input_partial_shape(0); + auto input_shape = input_pshape.to_shape(); + ov::Shape output_shape(input_shape); + set_output_type(0, get_input_element_type(0), ov::PartialShape(output_shape)); + } + + bool visit_attributes(AttributeVisitor& /*visitor*/) override; + + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override { + OPENVINO_ASSERT(new_args.size() == 1, "Incorrect number of new arguments"); + return std::make_shared(new_args.at(0)); + } +}; + +class FailGracefullyTest : + public ov::test::behavior::OVPluginTestBase, + public testing::WithParamInterface { +protected: + std::shared_ptr core = utils::PluginCache::get().core(); + ov::AnyMap configuration; + std::shared_ptr ov_model; + +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string targetDevice; + ov::AnyMap configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '.'); + + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + result << "targetPlatform=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(targetDevice) << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + + return result.str(); + } + + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + + SKIP_IF_CURRENT_TEST_IS_DISABLED() + OVPluginTestBase::SetUp(); + ov_model = createModelWithUnknownNode(); + } + + void TearDown() override { + if (!configuration.empty()) { + utils::PluginCache::get().reset(); + } + APIBaseTest::TearDown(); + } + +private: + std::shared_ptr createModelWithUnknownNode() { + const ov::Shape input_shape = {1, 4096}; + const ov::element::Type precision = ov::element::f32; + + ov::ParameterVector params{std::make_shared(precision, ov::Shape{input_shape})}; + auto constant = ov::test::utils::deprecated::make_constant(precision, {4096, 1024}, std::vector{}, true); + auto custom_op = std::make_shared(constant); + + ov::NodeVector results{custom_op}; + return std::make_shared(results, ov::ParameterVector{params}, "CustomOpModel"); + } +}; + +TEST_P(FailGracefullyTest, OnUnsupprotedOperator) { + auto compilerType = configuration[ov::intel_npu::compiler_type.name()].as(); + try { + core->compile_model(ov_model, target_device, configuration); + } catch (std::exception& ex) { + // TODO: the below error messages will be improved in E#64716 + if (compilerType == "MLIR") { + EXPECT_THAT(ex.what(), + AllOf(HasSubstr("Unsupported operation"), HasSubstr("with type UnsupportedTestOperation"))); + } else if (compilerType == "DRIVER") { + EXPECT_THAT(ex.what(), AllOf(HasSubstr("Failed to compile network"))); + } + return; + } + + ASSERT_FALSE(true) << "Oops, compilation of unsupported op happened"; +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/behavior/infer_request_run.cpp b/src/plugins/intel_npu/tests/functional/behavior/infer_request_run.cpp new file mode 100644 index 00000000000..0eb80ec7b5b --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/infer_request_run.cpp @@ -0,0 +1,29 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/infer_request_run.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" +#include "npu_private_properties.hpp" + +using namespace ov::test::behavior; + +const std::vector configsInferRequestRunTests = { + {ov::log::level(ov::log::Level::INFO)} +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTest, InferRequestRunTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configsInferRequestRunTests)), + InferRequestRunTests::getTestCaseName); + +const std::vector batchingConfigs = { + {ov::log::level(ov::log::Level::WARNING), ov::intel_npu::batch_mode(ov::intel_npu::BatchMode::COMPILER)}, + {ov::log::level(ov::log::Level::WARNING), ov::intel_npu::batch_mode(ov::intel_npu::BatchMode::AUTO)}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTest, BatchingRunTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(batchingConfigs)), + InferRequestRunTests::getTestCaseName); diff --git a/src/plugins/intel_npu/tests/functional/behavior/infer_request_run.hpp b/src/plugins/intel_npu/tests/functional/behavior/infer_request_run.hpp new file mode 100644 index 00000000000..25068c0fc07 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/infer_request_run.hpp @@ -0,0 +1,399 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +#include "npu_private_properties.hpp" + +#include "base/ov_behavior_test_utils.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "functional_test_utils/ov_plugin_cache.hpp" +#include "overload/overload_test_utils_npu.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using CompilationParams = std::tuple; + +using ::testing::AllOf; +using ::testing::HasSubstr; + +namespace ov { +namespace test { +namespace behavior { +class InferRequestRunTests : + public ov::test::behavior::OVPluginTestBase, + public testing::WithParamInterface { +protected: + std::shared_ptr core = utils::PluginCache::get().core(); + ov::AnyMap configuration; + std::shared_ptr ov_model; + ov::CompiledModel compiledModel; + ov::Output input; + ov::Output output; + std::string m_cache_dir; + +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string targetDevice; + ov::AnyMap configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '_'); + targetDevice = ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); + + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + result << "targetPlatform=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(targetDevice) << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + + return result.str(); + } + + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + + SKIP_IF_CURRENT_TEST_IS_DISABLED() + OVPluginTestBase::SetUp(); + ov_model = getDefaultNGraphFunctionForTheDeviceNPU(); // FIXME: E#80555 + } + + std::string generateCacheDirName(const std::string& test_name) { + using namespace std::chrono; + // Generate unique file names based on test name, thread id and timestamp + // This allows execution of tests in parallel (stress mode) + auto hash = std::to_string(std::hash()(test_name)); + std::stringstream ss; + auto ts = duration_cast(high_resolution_clock::now().time_since_epoch()); + ss << hash << "_" + << "_" << ts.count(); + return ss.str(); + } + + void TearDown() override { + if (!m_cache_dir.empty()) { + core->set_property({ov::cache_dir()}); + core.reset(); + ov::test::utils::PluginCache::get().reset(); + ov::test::utils::removeFilesWithExt(m_cache_dir, "blob"); + ov::test::utils::removeDir(m_cache_dir); + } + + if (!configuration.empty()) { + utils::PluginCache::get().reset(); + } + + APIBaseTest::TearDown(); + } + + std::shared_ptr createBatchingModel(element::Type type, const PartialShape& shape, + const ov::Layout& layout) { + ResultVector res; + ParameterVector params; + + auto data1 = std::make_shared(type, shape); + data1->set_friendly_name("input"); + data1->get_output_tensor(0).set_names({"tensor_input"}); + data1->set_layout(layout); + auto constant = opset8::Constant::create(type, {1}, {1}); + auto op1 = std::make_shared(data1, constant); + op1->set_friendly_name("Add"); + auto res1 = std::make_shared(op1); + res1->set_friendly_name("Result"); + res1->get_output_tensor(0).set_names({"tensor_output"}); + params.push_back(data1); + res.push_back(res1); + + return std::make_shared(res, params); + } +}; + +TEST_P(InferRequestRunTests, AllocatorCanDisposeBlobWhenOnlyInferRequestIsInScope) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + ov::InferRequest req; + ov::Tensor outputTensor; + { + core.reset(); + ov::test::utils::PluginCache::get().reset(); + } + } + std::cout << "Plugin should be unloaded from memory at this point" << std::endl; +} + +TEST_P(InferRequestRunTests, MultipleExecutorStreamsTestsSyncInfers) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + // Load CNNNetwork to target plugins + OV_ASSERT_NO_THROW(compiledModel = core->compile_model(ov_model, target_device, configuration)); + OV_ASSERT_NO_THROW(input = compiledModel.input()); + OV_ASSERT_NO_THROW(output = compiledModel.output()); + + // Create InferRequests + const int inferReqNumber = 256; + std::array inferReqs; + std::array inferReqsThreads; + for (int i = 0; i < inferReqNumber; ++i) { + OV_ASSERT_NO_THROW(inferReqs[i] = compiledModel.create_infer_request()); + OV_ASSERT_NO_THROW(inferReqs[i].get_tensor(input)); + } + + for (int i = 0; i < inferReqNumber; ++i) { + ov::InferRequest& infReq = inferReqs[i]; + inferReqsThreads[i] = std::thread([&infReq]() -> void { + OV_ASSERT_NO_THROW(infReq.infer()); + }); + } + + for (int i = 0; i < inferReqNumber; ++i) { + inferReqsThreads[i].join(); + OV_ASSERT_NO_THROW(inferReqs[i].get_tensor(output)); + } +} + +TEST_P(InferRequestRunTests, MultipleExecutorStreamsTestsAsyncInfers) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + // Load CNNNetwork to target plugins + OV_ASSERT_NO_THROW(compiledModel = core->compile_model(ov_model, target_device, configuration)); + OV_ASSERT_NO_THROW(input = compiledModel.input()); + OV_ASSERT_NO_THROW(output = compiledModel.output()); + + // Create InferRequests + const int inferReqNumber = 256; + std::array inferReqs; + for (int i = 0; i < inferReqNumber; ++i) { + OV_ASSERT_NO_THROW(inferReqs[i] = compiledModel.create_infer_request()); + OV_ASSERT_NO_THROW(inferReqs[i].get_tensor(input)); + } + + for (int i = 0; i < inferReqNumber; ++i) { + OV_ASSERT_NO_THROW(inferReqs[i].start_async()); + } + + for (int i = 0; i < inferReqNumber; ++i) { + inferReqs[i].wait(); + OV_ASSERT_NO_THROW(inferReqs[i].get_tensor(output)); + } +} + +TEST_P(InferRequestRunTests, MultipleExecutorTestsSyncInfers) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + // Load CNNNetwork to target plugins + OV_ASSERT_NO_THROW(compiledModel = core->compile_model(ov_model, target_device, configuration)); + OV_ASSERT_NO_THROW(input = compiledModel.input()); + OV_ASSERT_NO_THROW(output = compiledModel.output()); + + // Create InferRequests + const int inferReqNumber = 256; + ov::InferRequest inferReq; + ov::Tensor input_tensor; + for (int i = 0; i < inferReqNumber; ++i) { + OV_ASSERT_NO_THROW(inferReq = compiledModel.create_infer_request()); + OV_ASSERT_NO_THROW(input_tensor = inferReq.get_tensor(input)); + OV_ASSERT_NO_THROW(inferReq.set_input_tensor(input_tensor)); + OV_ASSERT_NO_THROW(inferReq.infer()); + OV_ASSERT_NO_THROW(inferReq.get_tensor(output)); + } +} + +TEST_P(InferRequestRunTests, CheckOutputDataFromTwoRuns) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + + ov::InferRequest inference_request; + ov::Tensor first_output; + ov::Tensor second_output; + float* data; + + { + OV_ASSERT_NO_THROW(compiledModel = core->compile_model(ov_model, target_device, configuration)); + OV_ASSERT_NO_THROW(inference_request = compiledModel.create_infer_request()); + auto tensor = inference_request.get_input_tensor(); + const size_t byte_size = tensor.get_byte_size(); + data = new float[byte_size / sizeof(float)]; + memset(data, 1, byte_size); + ov::Tensor input_data_tensor{ov::element::f32, tensor.get_shape(), data}; + OV_ASSERT_NO_THROW(inference_request.set_input_tensor(input_data_tensor)); + OV_ASSERT_NO_THROW(inference_request.infer()); + } + first_output = inference_request.get_output_tensor(0); + + for (int i = 0; i < 10; i++) { + delete[] data; + compiledModel = {}; + inference_request = {}; + + { + OV_ASSERT_NO_THROW(compiledModel = core->compile_model(ov_model, target_device, configuration)); + OV_ASSERT_NO_THROW(inference_request = compiledModel.create_infer_request()); + auto tensor = inference_request.get_input_tensor(); + const size_t byte_size = tensor.get_byte_size(); + data = new float[byte_size / sizeof(float)]; + memset(data, 1, byte_size); + ov::Tensor input_data_tensor{ov::element::f32, tensor.get_shape(), data}; + OV_ASSERT_NO_THROW(inference_request.set_input_tensor(input_data_tensor)); + OV_ASSERT_NO_THROW(inference_request.infer()); + } + second_output = inference_request.get_output_tensor(0); + + EXPECT_NE(first_output.data(), second_output.data()); + EXPECT_EQ(memcmp(first_output.data(), second_output.data(), second_output.get_byte_size()), 0); + } +} + +using BatchingRunTests = InferRequestRunTests; + +TEST_P(BatchingRunTests, CheckBatchingSupportInfer) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + + ov::InferRequest inference_request; + auto batch_shape = Shape{4, 2, 32, 32}; + std::shared_ptr ov_model_batch = createBatchingModel(element::f32, batch_shape, "N..."); + + OV_ASSERT_NO_THROW(compiledModel = core->compile_model(ov_model_batch, target_device, configuration)); + OV_ASSERT_NO_THROW(inference_request = compiledModel.create_infer_request()); + OV_ASSERT_NO_THROW(inference_request.infer()); +} + +TEST_P(BatchingRunTests, CheckBatchingSupportAsync) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + + ov::InferRequest inference_request; + auto batch_shape = Shape{4, 2, 32, 32}; + std::shared_ptr ov_model_batch = createBatchingModel(element::f32, batch_shape, "N..."); + + OV_ASSERT_NO_THROW(compiledModel = core->compile_model(ov_model_batch, target_device, configuration)); + OV_ASSERT_NO_THROW(inference_request = compiledModel.create_infer_request()); + OV_ASSERT_NO_THROW(inference_request.start_async()); + inference_request.wait(); +} + +TEST_P(BatchingRunTests, UseCompilerBatchingErrorPluginBatching) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + + ov::InferRequest inference_request; + std::shared_ptr ov_model_batch = getDefaultNGraphFunctionForTheDeviceNPU({4, 2, 32, 32}); + + auto batch_mode = configuration[ov::intel_npu::batch_mode.name()].as(); + try { + compiledModel = core->compile_model(ov_model_batch, target_device, configuration); + inference_request = compiledModel.create_infer_request(); + inference_request.start_async(); + inference_request.wait(); + } catch (std::exception& ex) { + if (batch_mode != "PLUGIN") { + ASSERT_FALSE(true) << ex.what(); + } + } +} + +TEST_P(BatchingRunTests, SetInputTensorInfer) { + auto batch_shape = Shape{4, 2, 2, 2}; + auto shape_size = ov::shape_size(batch_shape); + auto model = createBatchingModel(element::f32, batch_shape, "N..."); + float* buffer = new float[shape_size]; + + compiledModel = core->compile_model(model, target_device, configuration); + ov::InferRequest inference_request; + inference_request = compiledModel.create_infer_request(); + + ov::Tensor tensor{element::f32, batch_shape, buffer}; + + inference_request.set_input_tensor(tensor); + auto actual_tensor = inference_request.get_output_tensor(0); + auto* actual = actual_tensor.data(); + auto* input_data = tensor.data(); + for (size_t i = 0; i < shape_size; ++i) { + input_data[i] = 5.f; + } + + inference_request.infer(); // Adds '1' to each element + for (size_t i = 0; i < shape_size; ++i) { + EXPECT_NEAR(actual[i], 6.f, 1e-5) << "Expected=6, actual=" << actual[i] << " for index " << i; + } +} + +TEST_P(BatchingRunTests, SetInputTensorAsync) { + auto batch_shape = Shape{4, 2, 2, 2}; + auto shape_size = ov::shape_size(batch_shape); + auto model = createBatchingModel(element::f32, batch_shape, "N..."); + float* buffer = new float[shape_size]; + + compiledModel = core->compile_model(model, target_device, configuration); + ov::InferRequest inference_request; + inference_request = compiledModel.create_infer_request(); + + ov::Tensor tensor{element::f32, batch_shape, buffer}; + + inference_request.set_input_tensor(tensor); + auto actual_tensor = inference_request.get_output_tensor(0); + auto* actual = actual_tensor.data(); + auto* input_data = tensor.data(); + for (size_t i = 0; i < shape_size; ++i) { + input_data[i] = 5.f; + } + + inference_request.start_async(); // Adds '1' to each element + inference_request.wait_for(std::chrono::milliseconds(1000)); + for (size_t i = 0; i < shape_size; ++i) { + EXPECT_NEAR(actual[i], 6.f, 1e-5) << "Expected=6, actual=" << actual[i] << " for index " << i; + } +} + +TEST_P(BatchingRunTests, SetInputTensorInfer_Caching) { + auto batch_shape = Shape{4, 2, 2, 2}; + auto shape_size = ov::shape_size(batch_shape); + auto model = createBatchingModel(element::f32, batch_shape, "N..."); + float* buffer = new float[shape_size]; + + m_cache_dir = generateCacheDirName(GetTestName()); + core->set_property({ov::cache_dir(m_cache_dir)}); + auto compiled_model_no_cache = core->compile_model(model, target_device, configuration); + compiledModel = core->compile_model(model, target_device, configuration); + ov::InferRequest inference_request; + inference_request = compiledModel.create_infer_request(); + + ov::Tensor tensor{element::f32, batch_shape, buffer}; + + inference_request.set_input_tensor(tensor); + auto actual_tensor = inference_request.get_output_tensor(0); + auto* actual = actual_tensor.data(); + auto* input_data = tensor.data(); + for (size_t i = 0; i < shape_size; ++i) { + input_data[i] = 5.f; + } + + inference_request.infer(); // Adds '1' to each element + for (size_t i = 0; i < shape_size; ++i) { + EXPECT_NEAR(actual[i], 6.f, 1e-5) << "Expected=6, actual=" << actual[i] << " for index " << i; + } +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/downgrade_interpolate11.cpp b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/downgrade_interpolate11.cpp new file mode 100644 index 00000000000..394424b42fa --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/downgrade_interpolate11.cpp @@ -0,0 +1,19 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "downgrade_interpolate11.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; + +const std::vector configs = { + {{ov::intel_npu::compiler_type(ov::intel_npu::CompilerType::DRIVER)}}, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTest, DriverCompilerAdapterDowngradeInterpolate11TestNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + DriverCompilerAdapterDowngradeInterpolate11TestNPU::getTestCaseName); diff --git a/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/downgrade_interpolate11.hpp b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/downgrade_interpolate11.hpp new file mode 100644 index 00000000000..391eecc6c7e --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/downgrade_interpolate11.hpp @@ -0,0 +1,97 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "base/ov_behavior_test_utils.hpp" +#include "common/npu_test_env_cfg.hpp" + +#include "openvino/op/constant.hpp" +#include "openvino/op/interpolate.hpp" + +using CompilationParams = std::tuple; + +namespace ov { +namespace test { +namespace behavior { +class DriverCompilerAdapterDowngradeInterpolate11TestNPU : + public ov::test::behavior::OVPluginTestBase, + public testing::WithParamInterface { +protected: + std::shared_ptr core = utils::PluginCache::get().core(); + ov::AnyMap configuration; + std::shared_ptr ov_model; + +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string targetDevice; + ov::AnyMap configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '.'); + + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + result << "targetPlatform=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(targetDevice) << "_"; + result << "model=" + << "Interpolate11Model" + << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + return result.str(); + } + + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov_model = createInterpolate11Model(); + OVPluginTestBase::SetUp(); + } + + void TearDown() override { + if (!configuration.empty()) { + utils::PluginCache::get().reset(); + } + APIBaseTest::TearDown(); + } + +private: + std::shared_ptr createInterpolate11Model() { + using InterpolateAttrs = op::v11::Interpolate::InterpolateAttrs; + using InterpolateMode = op::v11::Interpolate::InterpolateMode; + using ShapeCalcMode = op::v11::Interpolate::ShapeCalcMode; + using TransformMode = op::v11::Interpolate::CoordinateTransformMode; + using NearestMode = op::v11::Interpolate::NearestMode; + const auto data = std::make_shared(ov::element::f32, ov::PartialShape{2, 2, 30, 60}); + const auto scales = + ov::op::v0::Constant::create(ov::element::f32, ov::Shape{2}, std::vector{0.5f, 0.5f}); + const auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, std::vector{2, 3}); + // Only modes of NEAREST, LINEAR, LINEAR_ONNX and CUBIC are supported for ConvertInterpolate11ToInterpolate4, + // here we use mode of NEAREST . + const InterpolateAttrs attrs{InterpolateMode::NEAREST, + ShapeCalcMode::SCALES, + std::vector{0, 0, 0, 0}, + std::vector{0, 0, 0, 0}, + TransformMode::HALF_PIXEL, + NearestMode::ROUND_PREFER_FLOOR, + false, + -0.75}; + const auto interpolate = std::make_shared(data, scales, axes, attrs); + ov::ResultVector results{std::make_shared(interpolate)}; + return std::make_shared(results, ov::ParameterVector{{data}}, "Interpolate-11"); + } +}; + +TEST_P(DriverCompilerAdapterDowngradeInterpolate11TestNPU, CheckOpsetVersion) { + EXPECT_NO_THROW(auto compiledModel = core->compile_model(ov_model, target_device, configuration);); +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/expected_throw.cpp b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/expected_throw.cpp new file mode 100644 index 00000000000..119c9c36e51 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/expected_throw.cpp @@ -0,0 +1,16 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "expected_throw.hpp" +#include "common/utils.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; + +const std::vector configs = {{{ov::intel_npu::compiler_type(ov::intel_npu::CompilerType::DRIVER)}}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTest, DriverCompilerAdapterExpectedThrowNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + DriverCompilerAdapterExpectedThrowNPU::getTestCaseName); diff --git a/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/expected_throw.hpp b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/expected_throw.hpp new file mode 100644 index 00000000000..b5e9885cea6 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/expected_throw.hpp @@ -0,0 +1,94 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "base/ov_behavior_test_utils.hpp" +#include "common/npu_test_env_cfg.hpp" + +using CompilationParams = std::tuple; + +namespace ov { +namespace test { +namespace behavior { +class DriverCompilerAdapterExpectedThrowNPU : + public ov::test::behavior::OVPluginTestBase, + public testing::WithParamInterface { +protected: + std::shared_ptr core = utils::PluginCache::get().core(); + ov::AnyMap configuration; + std::shared_ptr ov_model; + +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string targetDevice; + ov::AnyMap configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '_'); + + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + result << "targetPlatform=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(targetDevice) << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + + return result.str(); + } + + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + + SKIP_IF_CURRENT_TEST_IS_DISABLED() + OVPluginTestBase::SetUp(); + ov_model = getDefaultNGraphFunctionForTheDevice(); + ov::AnyMap params; + for (auto&& v : configuration) { + params.emplace(v.first, v.second); + } + } + + std::string getEnv(const char* name) { + const char* ret = getenv(name); + if (!ret) + return std::string(); + return std::string(ret); + } +}; + +TEST_P(DriverCompilerAdapterExpectedThrowNPU, CheckWrongGraphExtAndThrow) { +#if defined(NPU_PLUGIN_DEVELOPER_BUILD) + const char* name = "ADAPTER_MANUAL_CONFIG"; + std::string env_value = getEnv(name); + +#ifdef _WIN32 + _putenv_s(name, "WRONG_VERSION"); +#else + setenv(name, "WRONG_VERSION", 1); +#endif + + EXPECT_THROW(auto compiledModel = core->compile_model(ov_model, target_device, configuration);, std::exception); + +#ifdef _WIN32 + _putenv_s(name, env_value.c_str()); +#else + if (!env_value.empty()) { + setenv(name, env_value.c_str(), 1); + } else { + unsetenv(name); + } +#endif +#endif +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/inputs_outputs.cpp b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/inputs_outputs.cpp new file mode 100644 index 00000000000..ba5a4a2f999 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/inputs_outputs.cpp @@ -0,0 +1,19 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "inputs_outputs.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; + +const std::vector configs = { + {{ov::intel_npu::compiler_type(ov::intel_npu::CompilerType::DRIVER)}}, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTest, DriverCompilerAdapterInputsOutputsTestNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + DriverCompilerAdapterInputsOutputsTestNPU::getTestCaseName); diff --git a/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/inputs_outputs.hpp b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/inputs_outputs.hpp new file mode 100644 index 00000000000..117356fd1db --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/inputs_outputs.hpp @@ -0,0 +1,121 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "base/ov_behavior_test_utils.hpp" +#include "common/npu_test_env_cfg.hpp" + +#include "openvino/op/relu.hpp" + +using CompilationParams = std::tuple; + +namespace ov { +namespace test { +namespace behavior { + +class DriverCompilerAdapterInputsOutputsTestNPU : + public ov::test::behavior::OVPluginTestBase, + public testing::WithParamInterface { +protected: + std::shared_ptr core = utils::PluginCache::get().core(); + ov::AnyMap configuration; + std::shared_ptr ov_model; + +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string targetDevice; + ov::AnyMap configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '.'); + + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + result << "targetPlatform=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(targetDevice) << "_"; + result << "model=" + << "SimpleReluModel" + << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + return result.str(); + } + + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov_model = createSimpleReluModel(); + OVPluginTestBase::SetUp(); + } + + void TearDown() override { + if (!configuration.empty()) { + utils::PluginCache::get().reset(); + } + APIBaseTest::TearDown(); + } + +private: + std::shared_ptr createSimpleReluModel() { + auto arg0 = std::make_shared(ov::element::f32, ov::PartialShape{1}); + arg0->set_friendly_name("data"); + arg0->get_output_tensor(0).set_names({"input"}); + + auto relu = std::make_shared(arg0); + relu->set_friendly_name("relu"); + relu->get_output_tensor(0).set_names({"relu_res"}); + auto f = std::make_shared(relu, ov::ParameterVector{arg0}); + f->validate_nodes_and_infer_types(); + return f; + } +}; + +TEST_P(DriverCompilerAdapterInputsOutputsTestNPU, CheckInOutputs) { + auto compiledModel = core->compile_model(ov_model, target_device, configuration); + for (auto&& input : compiledModel.inputs()) { + std::string in_name; + std::string in_node_name; + try { + for (const auto& name : input.get_names()) { + in_name += name + " , "; + } + in_name = in_name.substr(0, in_name.size() - 3); + } catch (const ov::Exception&) { + } + + try { + in_node_name = input.get_node()->get_friendly_name(); + } catch (const ov::Exception&) { + } + EXPECT_EQ(in_name, "input"); + EXPECT_EQ(in_node_name, "data"); + } + + for (auto&& output : compiledModel.outputs()) { + std::string out_name; + std::string out_node_name; + + try { + for (const auto& name : output.get_names()) { + out_name += name + " , "; + } + out_name = out_name.substr(0, out_name.size() - 3); + } catch (const ov::Exception&) { + } + try { + out_node_name = output.get_node()->get_input_node_ptr(0)->get_friendly_name(); + } catch (const ov::Exception&) { + } + EXPECT_EQ(out_name, "relu_res"); + EXPECT_EQ(out_node_name, "relu"); + } +} +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/query_network.cpp b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/query_network.cpp new file mode 100644 index 00000000000..2f79791b9d3 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/query_network.cpp @@ -0,0 +1,170 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include "base/ov_behavior_test_utils.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "common_test_utils/node_builders/constant.hpp" +#include "common_test_utils/subgraph_builders/conv_pool_relu.hpp" +#include "common_test_utils/subgraph_builders/conv_pool_relu_non_zero.hpp" +#include "intel_npu/al/config/common.hpp" + +using CompilationParams = std::tuple; + +namespace ov { +namespace test { +namespace behavior { + +class UnsupportedTestOp : public ov::op::Op { +public: + OPENVINO_OP("UnsupportedTestOp"); + + UnsupportedTestOp() = default; + explicit UnsupportedTestOp(const ov::Output& arg): Op({arg}) { + constructor_validate_and_infer_types(); + } + + void validate_and_infer_types() override { + auto input_pshape = get_input_partial_shape(0); + auto input_shape = input_pshape.to_shape(); + ov::Shape output_shape(input_shape); + set_output_type(0, get_input_element_type(0), ov::PartialShape(output_shape)); + } + + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override { + if (new_args.size() != 1) { + OPENVINO_THROW("Incorrect number of new arguments"); + } + + return std::make_shared(new_args.at(0)); + } +}; + +std::shared_ptr createModelWithUnknownNode(); + +std::shared_ptr createModelWithUnknownNode() { + const ov::Shape input_shape = {1, 4096}; + const ov::element::Type precision = ov::element::f32; + ov::ParameterVector params = {std::make_shared(precision, ov::Shape{input_shape})}; + auto constant = ov::test::utils::deprecated::make_constant(precision, {4096, 1024}, std::vector{}, true); + auto custom_op = std::make_shared(constant); + + ov::NodeVector results{custom_op}; + return std::make_shared(results, ov::ParameterVector{params}, "CustomOpModel"); +} + +class QueryNetworkTestNPU : + public ov::test::behavior::OVPluginTestBase, + public testing::WithParamInterface { +public: + ov::SupportedOpsMap testQueryNetwork(std::shared_ptr model) { + std::shared_ptr core = utils::PluginCache::get().core(); + ov::AnyMap config; + return core->query_model(model, target_device, config); + } + + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + SKIP_IF_CURRENT_TEST_IS_DISABLED() + OVPluginTestBase::SetUp(); + } + + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string targetDevice; + ov::AnyMap configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '.'); + + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + result << "targetPlatform=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(targetDevice) << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + return result.str(); + } + + void TearDown() override { + if (!configuration.empty()) { + utils::PluginCache::get().reset(); + } + APIBaseTest::TearDown(); + } + +protected: + ov::AnyMap configuration; +}; + +const std::vector configs = {{}}; + +using QueryNetworkTestSuite1NPU = QueryNetworkTestNPU; + +// Test query with a supported OpenVINO model +TEST_P(QueryNetworkTestSuite1NPU, TestQueryNetworkSupported) { + const auto supportedModel = ov::test::utils::make_conv_pool_relu(); + ov::SupportedOpsMap result; + EXPECT_NO_THROW(result = testQueryNetwork(supportedModel)); + std::unordered_set expected, actual; + for (auto& op : supportedModel->get_ops()) { + expected.insert(op->get_friendly_name()); + } + for (auto& name : result) { + actual.insert(name.first); + } + EXPECT_EQ(expected, actual); +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTest, QueryNetworkTestSuite1NPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + QueryNetworkTestNPU::getTestCaseName); + +using QueryNetworkTestSuite2NPU = QueryNetworkTestNPU; + +// Test query with an unsupported ngraph function +// Tracking E#103192 +TEST_P(QueryNetworkTestSuite2NPU, DISABLED_TestQueryNetworkUnsupported) { + ov::SupportedOpsMap result; + const auto unsupportedModel = ov::test::utils::make_conv_pool_relu_non_zero(); + EXPECT_NO_THROW(result = testQueryNetwork(unsupportedModel)); + std::unordered_set expected, actual; + for (auto& op : unsupportedModel->get_ops()) { + expected.insert(op->get_friendly_name()); + } + for (auto& name : result) { + actual.insert(name.first); + } + if (actual.empty()) { + GTEST_SKIP() << "Skip the tests since QueryNetwork is unsupported with current driver"; + } else { + EXPECT_NE(expected, actual); + } +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTest, QueryNetworkTestSuite2NPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + QueryNetworkTestNPU::getTestCaseName); + +using QueryNetworkTestSuite3NPU = QueryNetworkTestNPU; + +// Test query with model containing an unknown Op +// Should throw error when calling testQueryNetwork, because it would fail at read_model in vcl +// TODO: E#86084, test would crash instead of throw error in Linux, need to fix that. Thus skip it for now. +TEST_P(QueryNetworkTestSuite3NPU, TestQueryNetworkThrow) { + ov::SupportedOpsMap result; + const auto unsupportedModel = createModelWithUnknownNode(); + EXPECT_ANY_THROW(result = testQueryNetwork(unsupportedModel)); +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/behavior/replace_sw_layers.cpp b/src/plugins/intel_npu/tests/functional/behavior/replace_sw_layers.cpp new file mode 100644 index 00000000000..54cc4c9e34d --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/replace_sw_layers.cpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include +#include "common/functions.h" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" +#include "npu_private_properties.hpp" +#include "openvino/op/clamp.hpp" +#include "openvino/runtime/intel_npu/properties.hpp" + +using CompileWithDummy_NPU3720 = ov::test::behavior::OVInferRequestTests; + +std::shared_ptr buildSingleLayerClampNetwork(); + +std::shared_ptr buildSingleLayerClampNetwork() { // Clamp is not supported in SW + ov::Shape inputShape = {1, 3, 4, 3}; + ov::element::Type netPrecision = ov::element::f32; + + const ov::ParameterVector params{std::make_shared(netPrecision, ov::Shape{inputShape})}; + + const auto clamp = std::make_shared(params.at(0), 0., 1.); + + const ov::ResultVector results{std::make_shared(clamp)}; + + auto ov_model = std::make_shared(results, params, "clamp"); + + return ov_model; +} + +namespace { + +TEST_P(CompileWithDummy_NPU3720, CompilationForSpecificPlatform) { + if (getBackendName(*core) == "LEVEL0") { + GTEST_SKIP() << "Skip due to failure on device"; + } + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + const auto& ov_model = buildSingleLayerClampNetwork(); + OV_ASSERT_NO_THROW(auto compiled_model = core->compile_model(ov_model, target_device, configuration)); + } +} + +const std::vector configs = { + {{ov::intel_npu::platform(ov::intel_npu::Platform::NPU3720)}, + {ov::intel_npu::compilation_mode_params("dummy-op-replacement=true")}}}; +// Must be successfully compiled with dummy-op-replacement=true + +INSTANTIATE_TEST_SUITE_P(smoke_precommit_BehaviorTest_Dummy, CompileWithDummy_NPU3720, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + CompileWithDummy_NPU3720::getTestCaseName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/behavior/work_with_devices.cpp b/src/plugins/intel_npu/tests/functional/behavior/work_with_devices.cpp new file mode 100644 index 00000000000..87c582bbbe0 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/work_with_devices.cpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/work_with_devices.hpp" +#include "common/utils.hpp" +#include "intel_npu/al/config/common.hpp" + +namespace { + +const std::vector configs = { + {{ov::log::level(ov::log::Level::DEBUG)}}, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTest, TestCompiledModelNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + TestCompiledModelNPU::getTestCaseName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/behavior/work_with_devices.hpp b/src/plugins/intel_npu/tests/functional/behavior/work_with_devices.hpp new file mode 100644 index 00000000000..c5fb948ee7e --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/work_with_devices.hpp @@ -0,0 +1,158 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include "common/functions.h" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" +#include "npu_private_properties.hpp" + +using CompilerType = ov::intel_npu::CompilerType; + +namespace { + +class TestCompiledModelNPU : + public ov::test::behavior::OVPluginTestBase, + public testing::WithParamInterface> { +public: + void SetUp() override { + std::tie(target_device, configuration) = GetParam(); + OVPluginTestBase::SetUp(); + } + + static std::string getTestCaseName(testing::TestParamInfo> obj) { + std::string targetDevice; + ov::AnyMap configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '.'); + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + result << "targetPlatform=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(targetDevice) << "_"; + if (!configuration.empty()) { + using namespace ov::test::utils; + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + return result.str(); + } + +protected: + ov::AnyMap configuration; + std::shared_ptr core = ov::test::utils::PluginCache::get().core(); +}; + +TEST_P(TestCompiledModelNPU, samePlatformProduceTheSameBlob) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + std::string platform = ov::test::utils::getTestsPlatformFromEnvironmentOr("3700"); + + configuration[ov::intel_npu::create_executor.name()] = "0"; + auto configuration1 = configuration; + configuration1[ov::intel_npu::platform.name()] = platform; + const auto& ov_model1 = buildSingleLayerSoftMaxNetwork(); + auto compiled_model1 = core->compile_model(ov_model1, target_device, configuration1); + std::stringstream blobStream1; + compiled_model1.export_model(blobStream1); + + auto configuration2 = configuration; + configuration2[ov::intel_npu::platform.name()] = platform; + const auto& ov_model2 = buildSingleLayerSoftMaxNetwork(); + auto compiled_model2 = core->compile_model(ov_model2, target_device, configuration2); + std::stringstream blobStream2; + compiled_model2.export_model(blobStream2); + + ASSERT_NE(0, blobStream1.str().size()); + ASSERT_EQ(0, std::memcmp(blobStream1.str().c_str(), blobStream2.str().c_str(), blobStream1.str().size())); + } +} + +class TestCompileModelWithoutDeviceNPU : public TestCompiledModelNPU { +protected: + void SetUp() override { + const auto devices = core->get_available_devices(); + const auto isNPUDeviceAvailable = + std::find_if(devices.cbegin(), devices.cend(), [this](const std::string& device) { + return device.find(target_device) != std::string::npos; + }) != devices.cend(); + if (isNPUDeviceAvailable) { + GTEST_SKIP() << "Skip the tests since device is available"; + } + } +}; + +TEST_P(TestCompileModelWithoutDeviceNPU, ThrowIfNoDeviceAndNoPlatform) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + const auto& ov_model = buildSingleLayerSoftMaxNetwork(); + ASSERT_THROW(auto compiled_model = core->compile_model(ov_model, target_device, configuration), ov::Exception); + } +} + +TEST_P(TestCompileModelWithoutDeviceNPU, NoThrowIfNoDeviceAndButPlatformPassed) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + auto netConfiguration = configuration; + netConfiguration[ov::intel_npu::platform.name()] = ov::test::utils::getTestsPlatformFromEnvironmentOr("3700"); + const auto& ov_model = buildSingleLayerSoftMaxNetwork(); + OV_ASSERT_NO_THROW(auto compiled_model = core->compile_model(ov_model, target_device, netConfiguration)); + } +} + +const std::map> wrongDevice = { + // {orig, {wrong for MLIR}} + {"VPU3700", {"VPU0000"}}, {"VPU3720", {"VPU0000"}}, +}; + +std::string getWrongDevice(const std::string_view platform, const CompilerType&) { + // here we mix up devices in order to test the check on the runtime side + auto device = wrongDevice.find(platform); + + if (device == wrongDevice.end()) { + OPENVINO_THROW("Cannot map wrong device for the platform ", platform); + } + return std::string(device->second[0]); +} + +const std::map> validDevice = { + // {orig, {valid for MLIR}} + {"VPU3700", {"VPU3700"}}, {"VPU3720", {"VPU3720"}} +}; + +std::string getValidDevice(const std::string_view platform, const CompilerType&) { + auto device = validDevice.find(platform); + + if (device == validDevice.end()) { + OPENVINO_THROW("Cannot map valid device for the platform ", platform); + } + return std::string(device->second[0]); +} + +TEST_P(TestCompileModelWithoutDeviceNPU, CheckDeviceInBlob) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + // Compile model to target plugins, wrong platform specified -> expect an exception + auto netConfigurationMLIR_wrong = configuration; + netConfigurationMLIR_wrong[ov::intel_npu::platform.name()] = + getWrongDevice(PlatformEnvironment::PLATFORM, CompilerType::MLIR); + netConfigurationMLIR_wrong[ov::intel_npu::compiler_type.name()] = "MLIR"; + const auto& ov_model1 = buildSingleLayerSoftMaxNetwork(); + EXPECT_ANY_THROW(auto compiled_model = + core->compile_model(ov_model1, target_device, netConfigurationMLIR_wrong)); + + // Compile model to target plugins, valid platform specified -> expect no exception + auto netConfigurationMLIR_valid = configuration; + netConfigurationMLIR_valid[ov::intel_npu::platform.name()] = + getValidDevice(PlatformEnvironment::PLATFORM, CompilerType::MLIR); + netConfigurationMLIR_valid[ov::intel_npu::compiler_type.name()] = "MLIR"; + const auto& ov_model2 = buildSingleLayerSoftMaxNetwork(); + EXPECT_NO_THROW(auto compiled_model = + core->compile_model(ov_model2, target_device, netConfigurationMLIR_valid)); + } +} + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/common/functions.cpp b/src/plugins/intel_npu/tests/functional/common/functions.cpp new file mode 100644 index 00000000000..0b22a8a89be --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/functions.cpp @@ -0,0 +1,37 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "functions.h" +#include "common/npu_test_env_cfg.hpp" +#include "openvino/op/softmax.hpp" +#include "openvino/runtime/intel_npu/properties.hpp" + +std::shared_ptr buildSingleLayerSoftMaxNetwork() { + ov::Shape inputShape = {1, 3, 4, 3}; + ov::element::Type model_type = ov::element::f32; + size_t axis = 1; + + const ov::ParameterVector params{std::make_shared(model_type, ov::Shape({inputShape}))}; + params.at(0)->set_friendly_name("Parameter"); + + const auto softMax = std::make_shared(params.at(0), axis); + softMax->set_friendly_name("softMax"); + + const ov::ResultVector results{std::make_shared(softMax)}; + results.at(0)->set_friendly_name("Result"); + + auto ov_model = std::make_shared(results, params, "softMax"); + + return ov_model; +} + +const std::string PlatformEnvironment::PLATFORM = []() -> std::string { + const auto& var = ov::test::utils::NpuTestEnvConfig::getInstance().IE_NPU_TESTS_PLATFORM; + if (!var.empty()) { + return var; + } else { + std::cerr << "Environment variable is not set: IE_NPU_TESTS_PLATFORM! Exiting..." << std::endl; + exit(-1); + } +}(); diff --git a/src/plugins/intel_npu/tests/functional/common/functions.h b/src/plugins/intel_npu/tests/functional/common/functions.h new file mode 100644 index 00000000000..5f4c91bdf6a --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/functions.h @@ -0,0 +1,16 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +// create dummy network for tests +std::shared_ptr buildSingleLayerSoftMaxNetwork(); + +// class encapsulated Platform getting from environmental variable +class PlatformEnvironment { +public: + static const std::string PLATFORM; +}; diff --git a/src/plugins/intel_npu/tests/functional/common/npu_test_env_cfg.cpp b/src/plugins/intel_npu/tests/functional/common/npu_test_env_cfg.cpp new file mode 100644 index 00000000000..8a2fd6039da --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/npu_test_env_cfg.cpp @@ -0,0 +1,208 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/config.hpp" +#include "intel_npu/al/config/common.hpp" + +#include +#include + +namespace ov::test::utils { + +NpuTestEnvConfig::NpuTestEnvConfig() { + // start reading obsolete environment variables + if (auto var = std::getenv("IE_KMB_TESTS_DEVICE_NAME")) { + IE_NPU_TESTS_DEVICE_NAME = var; + } + + if (auto var = std::getenv("IE_KMB_TESTS_DUMP_PATH")) { + IE_NPU_TESTS_DUMP_PATH = var; + } + + if (auto var = std::getenv("IE_KMB_TESTS_LOG_LEVEL")) { + IE_NPU_TESTS_LOG_LEVEL = var; + } + + if (auto var = std::getenv("IE_KMB_TESTS_RUN_COMPILER")) { + IE_NPU_TESTS_RUN_COMPILER = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_RUN_COMPILER", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_RUN_EXPORT")) { + IE_NPU_TESTS_RUN_EXPORT = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_RUN_EXPORT", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_RUN_IMPORT")) { + IE_NPU_TESTS_RUN_IMPORT = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_RUN_IMPORT", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_RUN_INFER")) { + IE_NPU_TESTS_RUN_INFER = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_RUN_INFER", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_EXPORT_INPUT")) { + IE_NPU_TESTS_EXPORT_INPUT = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_EXPORT_INPUT", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_EXPORT_OUTPUT")) { + IE_NPU_TESTS_EXPORT_OUTPUT = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_EXPORT_OUTPUT", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_EXPORT_REF")) { + IE_NPU_TESTS_EXPORT_REF = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_EXPORT_REF", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_IMPORT_INPUT")) { + IE_NPU_TESTS_IMPORT_INPUT = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_IMPORT_INPUT", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_IMPORT_REF")) { + IE_NPU_TESTS_IMPORT_REF = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_IMPORT_REF", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_RAW_EXPORT")) { + IE_NPU_TESTS_RAW_EXPORT = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_RAW_EXPORT", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_LONG_FILE_NAME")) { + IE_NPU_TESTS_LONG_FILE_NAME = ::intel_npu::envVarStrToBool("IE_KMB_TESTS_LONG_FILE_NAME", var); + } + + if (auto var = std::getenv("IE_KMB_TESTS_PLATFORM")) { + IE_NPU_TESTS_PLATFORM = var; + } + // end reading obsolete environment variables + + if (auto var = std::getenv("IE_NPU_TESTS_DEVICE_NAME")) { + IE_NPU_TESTS_DEVICE_NAME = var; + } + + if (auto var = std::getenv("IE_NPU_TESTS_DUMP_PATH")) { + IE_NPU_TESTS_DUMP_PATH = var; + } + + if (auto var = std::getenv("IE_NPU_TESTS_LOG_LEVEL")) { + IE_NPU_TESTS_LOG_LEVEL = var; + } + + if (auto var = std::getenv("IE_NPU_TESTS_RUN_COMPILER")) { + IE_NPU_TESTS_RUN_COMPILER = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_RUN_COMPILER", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_RUN_EXPORT")) { + IE_NPU_TESTS_RUN_EXPORT = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_RUN_EXPORT", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_RUN_IMPORT")) { + IE_NPU_TESTS_RUN_IMPORT = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_RUN_IMPORT", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_RUN_INFER")) { + IE_NPU_TESTS_RUN_INFER = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_RUN_INFER", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_EXPORT_INPUT")) { + IE_NPU_TESTS_EXPORT_INPUT = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_EXPORT_INPUT", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_EXPORT_OUTPUT")) { + IE_NPU_TESTS_EXPORT_OUTPUT = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_EXPORT_OUTPUT", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_EXPORT_REF")) { + IE_NPU_TESTS_EXPORT_REF = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_EXPORT_REF", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_IMPORT_INPUT")) { + IE_NPU_TESTS_IMPORT_INPUT = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_IMPORT_INPUT", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_IMPORT_REF")) { + IE_NPU_TESTS_IMPORT_REF = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_IMPORT_REF", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_RAW_EXPORT")) { + IE_NPU_TESTS_RAW_EXPORT = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_RAW_EXPORT", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_LONG_FILE_NAME")) { + IE_NPU_TESTS_LONG_FILE_NAME = ::intel_npu::envVarStrToBool("IE_NPU_TESTS_LONG_FILE_NAME", var); + } + + if (auto var = std::getenv("IE_NPU_TESTS_PLATFORM")) { + IE_NPU_TESTS_PLATFORM = var; + } +} + +const NpuTestEnvConfig& NpuTestEnvConfig::getInstance() { + static NpuTestEnvConfig instance{}; + return instance; +} + +std::string getTestsDeviceNameFromEnvironmentOr(const std::string& instead) { + return (!NpuTestEnvConfig::getInstance().IE_NPU_TESTS_DEVICE_NAME.empty()) + ? NpuTestEnvConfig::getInstance().IE_NPU_TESTS_DEVICE_NAME + : instead; +} + +std::string getTestsPlatformFromEnvironmentOr(const std::string& instead) { + return (!NpuTestEnvConfig::getInstance().IE_NPU_TESTS_PLATFORM.empty()) + ? NpuTestEnvConfig::getInstance().IE_NPU_TESTS_PLATFORM + : instead; +} + +std::string getDeviceNameTestCase(const std::string& str) { + ov::DeviceIDParser parser = ov::DeviceIDParser(str); + return parser.get_device_name() + parser.get_device_id(); +} + +std::string getDeviceName() { + return ov::test::utils::getTestsDeviceNameFromEnvironmentOr("NPU.3700"); +} + +std::string getDeviceNameID(const std::string& str) { + ov::DeviceIDParser parser = ov::DeviceIDParser(str); + return parser.get_device_id(); +} + +} // namespace ov::test::utils + +namespace InferRequestParamsAnyMapTestName { +std::string getTestCaseName(testing::TestParamInfo obj) { + std::string targetDevice; + ov::AnyMap configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '.'); + targetDevice = ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + result << "_"; + } + } + return result.str(); +} +} // namespace InferRequestParamsAnyMapTestName + +namespace InferRequestParamsMapTestName { + +std::string getTestCaseName(testing::TestParamInfo obj) { + std::string targetDevice; + std::map configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '.'); + targetDevice = ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_" << configItem.second << "_"; + } + } + return result.str(); +} +} // namespace InferRequestParamsMapTestName diff --git a/src/plugins/intel_npu/tests/functional/common/npu_test_env_cfg.hpp b/src/plugins/intel_npu/tests/functional/common/npu_test_env_cfg.hpp new file mode 100644 index 00000000000..a2281f9407d --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/npu_test_env_cfg.hpp @@ -0,0 +1,83 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "base/ov_behavior_test_utils.hpp" +#include "common/utils.hpp" +#include "npu_private_properties.hpp" + +#include +#include +#include + +using namespace ov::test::behavior; + +namespace ov::test::utils { + +/** + * Reads configuration environment variables + */ +class NpuTestEnvConfig { +public: + std::string IE_NPU_TESTS_DEVICE_NAME; + std::string IE_NPU_TESTS_DUMP_PATH; + std::string IE_NPU_TESTS_LOG_LEVEL; + std::string IE_NPU_TESTS_PLATFORM; + + bool IE_NPU_TESTS_RUN_COMPILER = true; + bool IE_NPU_TESTS_RUN_EXPORT = false; + bool IE_NPU_TESTS_RUN_IMPORT = false; + bool IE_NPU_TESTS_RUN_INFER = true; + bool IE_NPU_TESTS_EXPORT_INPUT = false; + bool IE_NPU_TESTS_EXPORT_OUTPUT = false; + bool IE_NPU_TESTS_EXPORT_REF = false; + bool IE_NPU_TESTS_IMPORT_INPUT = false; + bool IE_NPU_TESTS_IMPORT_REF = false; + + bool IE_NPU_TESTS_RAW_EXPORT = false; + bool IE_NPU_TESTS_LONG_FILE_NAME = false; + +public: + static const NpuTestEnvConfig& getInstance(); + +private: + explicit NpuTestEnvConfig(); +}; + +std::string getTestsDeviceNameFromEnvironmentOr(const std::string& instead); +std::string getTestsPlatformFromEnvironmentOr(const std::string& instead); + +std::string getDeviceNameTestCase(const std::string& str); +std::string getDeviceName(); +std::string getDeviceNameID(const std::string& str); + +// Current version of gtest seems to fail when parsing template functions for getTestCaseName +// To overcome this issue, programmer must make sure to wrap in paratheses this function when +// it is given at INSTANTIATE_TEST_SUITE_P macros +// e.g. INSTANTIATE_TEST_SUITE_P(TEST_SUITE_NAME, TEST_SUITE_CLASS, params, +// (appendPlatformTypeTestName)) +template +std::string appendPlatformTypeTestName(testing::TestParamInfo obj) { + const std::string& test_name = GenericTestCaseNameClass::getTestCaseName(obj); + return test_name + "_targetPlatform=" + getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); +} + +} // namespace ov::test::utils + +namespace InferRequestParamsAnyMapTestName { + +std::string getTestCaseName(testing::TestParamInfo obj); + +} // namespace InferRequestParamsAnyMapTestName + +namespace InferRequestParamsMapTestName { + +typedef std::tuple // Config + > + InferRequestParams; +std::string getTestCaseName(testing::TestParamInfo obj); + +} // namespace InferRequestParamsMapTestName diff --git a/src/plugins/intel_npu/tests/functional/common/print_test_case_name.hpp b/src/plugins/intel_npu/tests/functional/common/print_test_case_name.hpp new file mode 100644 index 00000000000..ad508c13e2a --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/print_test_case_name.hpp @@ -0,0 +1,37 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +// Used as a custom test case printer for the google test. +// It will transform a string like "Normalized: true, NmsThreshold: 0.45" +// into "Normalized=true_NmsThreshold=0.45" +// to have it as a test case name. +// This parameter string will be printed when the test fails. +struct PrintTestCaseName { + template + std::string operator()(const ::testing::TestParamInfo& info) const { + auto paramsString = ::testing::PrintToString(info.param); + + const auto allowedChar = [](const unsigned char c) { + return std::isalnum(c) || c == '_' || c == '.' || c == ':' || c == ','; + }; + const auto newEnd = std::remove_if(paramsString.begin(), paramsString.end(), std::not_fn(allowedChar)); + paramsString.erase(newEnd, paramsString.end()); + + const auto isColon = [](const unsigned char c) { + return c == ':'; + }; + std::replace_if(paramsString.begin(), paramsString.end(), isColon, '='); + + const auto isComma = [](const unsigned char c) { + return c == ','; + }; + std::replace_if(paramsString.begin(), paramsString.end(), isComma, '_'); + + return paramsString; + } +}; diff --git a/src/plugins/intel_npu/tests/functional/common/random_generator.cpp b/src/plugins/intel_npu/tests/functional/common/random_generator.cpp new file mode 100644 index 00000000000..96585887828 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/random_generator.cpp @@ -0,0 +1,14 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "random_generator.hpp" + +RandomGenerator& RandomGenerator::getInstance() { + static RandomGenerator instance; + return instance; +} + +std::mt19937& RandomGenerator::get() { + return getInstance().generator; +} diff --git a/src/plugins/intel_npu/tests/functional/common/random_generator.hpp b/src/plugins/intel_npu/tests/functional/common/random_generator.hpp new file mode 100644 index 00000000000..71f02926998 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/random_generator.hpp @@ -0,0 +1,22 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +class RandomGenerator { +public: + static std::mt19937& get(); + +private: + static RandomGenerator& getInstance(); + + RandomGenerator() = default; + RandomGenerator(const RandomGenerator&) = delete; + RandomGenerator& operator=(const RandomGenerator&) = delete; + ~RandomGenerator() = default; + + std::mt19937 generator{std::random_device{}()}; +}; diff --git a/src/plugins/intel_npu/tests/functional/common/tensor_view.hpp b/src/plugins/intel_npu/tests/functional/common/tensor_view.hpp new file mode 100644 index 00000000000..db854e6b6de --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/tensor_view.hpp @@ -0,0 +1,128 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +// Creates a wrapper around ov::Tensor and provides multi-dimensional checked element access +// +// Usage0: const auto tensor = ov::Tensor(ov::element::f32, ov::Shape{1, 24}); +// const auto tensorView = TensorView(tensor); +// const float data = tensorView.at(n, c); +// Provided tensor must have number of dimensions equal to the template argument (checked at the runtime) +// Indices are bound-checked on each access +// +// Usage1: struct Box { float x0, y0, x1, y1; }; +// const auto tensor = ov::Tensor(ov::element::f32, ov::Shape{C, H, W})); +// const auto boxesShape = ov::Shape{C, H, W / 4}; +// const auto boxesView = TensorView(tensor, boxesShape); +// const Box box = boxesView.at(c, h, w); // 0 <= w < W / 4 +// Interpret underlying data as a custom type. +// Inner dimension must be divisible by size of a template class (Box). +// +template +class TensorView { +private: + TensorView(ElementType* data, ov::Shape shape): _data{data}, _shape{std::move(shape)} { + VPUX_THROW_UNLESS(_data != nullptr, "Tensor data pointer is null"); + } + +public: + explicit TensorView(const ov::Tensor& tensor) { + init(tensor); + } + + explicit TensorView(const ov::Tensor& tensor, const ov::Shape& shape) { + init(tensor, &shape); + } + + // the secret to accessing private realms? self-friendship, of course + template + friend class TensorView; + + template + TensorView reshape(ov::Shape shape) const { + VPUX_THROW_UNLESS(shape.size() == NewDimSize, "Reshaping to {0}D tensor view, but {1}D shape provided", + NewDimSize, shape.size()); + VPUX_THROW_UNLESS(shape_size(this->_shape) == shape_size(shape), + "Failed to reshape tensor with shape {0} to shape {1}", this->_shape.to_string(), + shape.to_string()); + return TensorView(_data, std::move(shape)); + } + + const ov::Shape& getShape() const { + return _shape; + } + + ElementType* data() const { + return _data; + } + + size_t size() const { + return ov::shape_size(_shape); + } + + template = 0> + ElementType& at(Indices&&... indices) { + return getValue(std::forward(indices)...); + } + + template = 0> + ElementType& at(Indices&&... indices) const { + return getValue(std::forward(indices)...); + } + +private: + void init(const ov::Tensor& tensor, const ov::Shape* shape = nullptr) { + static_assert(!std::is_pointer_v, "Underlying type must not be a pointer"); + + _data = reinterpret_cast(tensor.data()); + _shape = tensor.get_shape(); + + const auto typeSizeRatio = sizeof(ElementType) / tensor.get_element_type().size(); + VPUX_THROW_UNLESS(typeSizeRatio > 0, "Underlying type size={0} is greater than sizeof({1})={2}", + tensor.get_element_type().size(), typeid(ElementType).name(), sizeof(ElementType)); + + VPUX_THROW_UNLESS(_shape.back() % typeSizeRatio == 0, + "Inner dimension of size={0} with sizeof({1})={2} is not divisible by sizeof({3})={4}", + _shape.back(), tensor.get_element_type().get_type_name(), tensor.get_element_type().size(), + typeid(ElementType).name(), sizeof(ElementType)); + _shape.back() /= typeSizeRatio; + + if (shape) { + *this = this->reshape(*shape); + } + + VPUX_THROW_UNLESS(_shape.size() == DimSize, "Creating {0}D tensor view for {1}D tensor", DimSize, + _shape.size()); + + _strides.back() = 1; + std::partial_sum(_shape.rbegin(), _shape.rend() - 1, _strides.rbegin() + 1, std::multiplies<>()); + } + + template + ElementType& getValue(Dims... dims) const { + static_assert(sizeof...(dims) == DimSize, "Invalid number of indices"); + + auto dimsArray = std::array{dims...}; + int index = 0; + + for (auto i = static_cast(dimsArray.size() - 1); i >= 0; i--) { + VPUX_THROW_UNLESS((dimsArray[i] >= 0 && dimsArray[i] < _shape[i]), + "Index {0} with value {1} is out of bounds=[{2}..{3})", i, dimsArray[i], 0, + static_cast(_shape[i])); + index += dimsArray[i] * _strides[i]; + } + + return _data[index]; + } + +private: + ElementType* _data; + ov::Shape _shape; + std::array _strides; +}; diff --git a/src/plugins/intel_npu/tests/functional/common/utils.cpp b/src/plugins/intel_npu/tests/functional/common/utils.cpp new file mode 100644 index 00000000000..1fbbdbebd66 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/utils.cpp @@ -0,0 +1,107 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "npu_private_properties.hpp" +#include "utils.hpp" + +namespace ov { + +namespace test { + +namespace utils { + +const char* DEVICE_NPU = "NPU"; + +} // namespace utils + +} // namespace test + +} // namespace ov + +std::string getBackendName(const ov::Core& core) { + return core.get_property("NPU", ov::intel_npu::backend_name.name()).as(); +} + +std::vector getAvailableDevices(const ov::Core& core) { + return core.get_property("NPU", ov::available_devices.name()).as>(); +} + +std::string modelPriorityToString(const ov::hint::Priority priority) { + std::ostringstream stringStream; + + stringStream << priority; + + return stringStream.str(); +} + +std::string removeDeviceNameOnlyID(const std::string& device_name_id) { + std::string::const_iterator first_digit = device_name_id.cend(); + std::string::const_iterator last_digit = device_name_id.cend(); + for (auto&& it = device_name_id.cbegin(); it != device_name_id.cend(); ++it) { + if (*it >= '0' && *it <= '9') { + if (first_digit == device_name_id.cend()) { + first_digit = it; + } + last_digit = it; + } + } + if (first_digit == device_name_id.cend()) { + return std::string(""); + } + return std::string(first_digit, last_digit + 1); +} + +std::vector getRWMandatoryPropertiesValues(std::vector props) { + ov::hint::PerformanceMode performanceModes[] = {ov::hint::PerformanceMode::LATENCY, + ov::hint::PerformanceMode::THROUGHPUT, + ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT}; + for (auto& performanceMode : performanceModes) { + if (std::find(props.begin(), props.end(), + ov::AnyMap{{ov::hint::performance_mode.name(), ov::Any(performanceMode)}}) != props.end()) { + continue; + } + props.push_back({{ov::hint::performance_mode(performanceMode)}}); + } + + if (std::find(props.begin(), props.end(), ov::AnyMap{{ov::hint::num_requests.name(), ov::Any(1)}}) == props.end()) { + props.push_back({{ov::hint::num_requests(1)}}); + } + + ov::hint::ExecutionMode executionModes[] = {ov::hint::ExecutionMode::PERFORMANCE, + ov::hint::ExecutionMode::ACCURACY}; + for (auto& executionMode : executionModes) { + if (std::find(props.begin(), props.end(), + ov::AnyMap{{ov::hint::execution_mode.name(), ov::Any(executionMode)}}) != props.end()) { + continue; + } + props.push_back({{ov::hint::execution_mode(executionMode)}}); + } + + bool enableProfilingValues[] = {true, false}; + for (auto enableProfilingValue : enableProfilingValues) { + if (std::find(props.begin(), props.end(), + ov::AnyMap{{ov::enable_profiling.name(), ov::Any(enableProfilingValue)}}) != props.end()) { + continue; + } + props.push_back({{ov::enable_profiling(enableProfilingValue)}}); + } + + ov::log::Level logLevels[] = {ov::log::Level::NO, ov::log::Level::ERR, ov::log::Level::WARNING, + ov::log::Level::INFO, ov::log::Level::DEBUG, ov::log::Level::TRACE}; + for (auto& logLevel : logLevels) { + if (std::find(props.begin(), props.end(), ov::AnyMap{{ov::log::level.name(), ov::Any(logLevel)}}) != + props.end()) { + continue; + } + props.push_back({ov::log::level(logLevel)}); + } + + if (std::find(props.begin(), props.end(), ov::AnyMap{{ov::streams::num.name(), ov::Any(ov::streams::num(3))}}) != + props.end()) { + props.push_back({ov::streams::num(3)}); + } + return props; +} diff --git a/src/plugins/intel_npu/tests/functional/common/utils.hpp b/src/plugins/intel_npu/tests/functional/common/utils.hpp new file mode 100644 index 00000000000..3616f81d71a --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/common/utils.hpp @@ -0,0 +1,91 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include "common_test_utils/unicode_utils.hpp" + +namespace ov { + +namespace test { + +namespace utils { + +extern const char* DEVICE_NPU; + +} // namespace utils + +} // namespace test + +} // namespace ov + +std::string getBackendName(const ov::Core& core); + +std::vector getAvailableDevices(const ov::Core& core); + +std::string modelPriorityToString(const ov::hint::Priority priority); + +std::string removeDeviceNameOnlyID(const std::string& device_name_id); + +std::vector getRWMandatoryPropertiesValues(std::vector props); + +template ::value || std::is_same::value)>::type> +void removeDirFilesRecursive(const std::basic_string& path) { + if (!ov::util::directory_exists(path)) { + return; + } + for (const auto& entry : std::filesystem::directory_iterator(path)) { + ov::test::utils::removeFile(entry.path().generic_string()); + } + ov::test::utils::removeDir(path); + // EISW-105043: [Linux] [Bug] Cannot delete loaded shared libraries unicode directories + // `Directory not empty` throw on linux for code below + // std::filesystem::remove_all(path); +} + +template +std::string vectorToString(std::vector v) { + std::ostringstream res; + for (size_t i = 0; i < v.size(); ++i) { + if (i != 0) { + res << ","; + } else { + res << "{"; + } + res << v[i]; + } + res << "}"; + return res.str(); +} + +// This utility class comes handy when OpenVino doesn't provide expicit `getTestCaseName` +// functions for declared test classes, but on NPU plugin side, we still need to append +// `_targetPlatform=NPUXXXX` to test name for activation of platform specific tests associated with test classes. +// Stands as a wrapper on `getTestCaseName` functions if declared and implements generic test name functions otherwise +// Programmer should NOT use this explicitly as it is part of `appendPlatformTypeTestName` utility. + +struct GenericTestCaseNameClass { + template + static constexpr bool hasGetTestCaseName = false; + + template + static std::string getTestCaseName(testing::TestParamInfo& obj) { + if constexpr (hasGetTestCaseName) { + return T::getTestCaseName(obj); + } else { + std::ostringstream result; + ::testing::PrintToStringParamName printToStringParamName; + result << printToStringParamName(obj); + return result.str(); + } + } +}; + +template +constexpr bool GenericTestCaseNameClass::hasGetTestCaseName< + T, std::void_t().getTestCaseName( + std::declval>()))>> = true; diff --git a/src/plugins/intel_npu/tests/functional/core_config.cpp b/src/plugins/intel_npu/tests/functional/core_config.cpp new file mode 100644 index 00000000000..eaa82c3f272 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/core_config.cpp @@ -0,0 +1,14 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "shared_test_classes/base/ov_subgraph.hpp" + +namespace ov { +namespace test { + +void core_configuration(ov::test::SubgraphBaseTest* test) { +} + +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/compile_and_infer.hpp b/src/plugins/intel_npu/tests/functional/internal/overload/compile_and_infer.hpp new file mode 100644 index 00000000000..c8728650201 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/internal/overload/compile_and_infer.hpp @@ -0,0 +1,84 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "base/ov_behavior_test_utils.hpp" +#include "openvino/opsets/opset8.hpp" + +namespace ov { +namespace test { +namespace behavior { + +inline std::shared_ptr getConstantGraph(element::Type type) { + ResultVector results; + ParameterVector params; + auto op = std::make_shared(opset8::Constant::create(type, {1}, {1}), + opset8::Constant::create(type, {1}, {1})); + op->set_friendly_name("Add"); + auto res = std::make_shared(op); + res->set_friendly_name("Result"); + res->get_output_tensor(0).set_names({"tensor_output"}); + results.push_back(res); + return std::make_shared(results, params); +} + +typedef std::tuple, // Model + std::string, // Device name + ov::AnyMap // Config + > + CompileAndInferRequestParams; + +class OVCompileAndInferRequest : + public testing::WithParamInterface, + public OVInferRequestTestBase { +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::shared_ptr model; + std::string targetDevice; + ov::AnyMap configuration; + std::tie(model, targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '.'); + std::ostringstream result; + result << "targetDevice=" << targetDevice << "_"; + if (!configuration.empty()) { + using namespace ov::test::utils; + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + return result.str(); + } + void SetUp() override { + std::tie(function, target_device, configuration) = this->GetParam(); + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + APIBaseTest::SetUp(); + } + +protected: + ov::CompiledModel execNet; + std::shared_ptr core = utils::PluginCache::get().core(); + std::string targetDevice; + ov::AnyMap configuration; + std::shared_ptr function; +}; + +TEST_P(OVCompileAndInferRequest, AsyncInferRequest) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(execNet = core->compile_model(function, target_device, configuration)); + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + bool is_called = false; + OV_ASSERT_NO_THROW(req.set_callback([&](std::exception_ptr exception_ptr) { + ASSERT_EQ(exception_ptr, nullptr); + is_called = true; + })); + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); + ASSERT_TRUE(is_called); +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/compiled_model/property.cpp b/src/plugins/intel_npu/tests/functional/internal/overload/compiled_model/property.cpp new file mode 100644 index 00000000000..3f58d7a7bcb --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/internal/overload/compiled_model/property.cpp @@ -0,0 +1,333 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/compiled_model/properties.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "common_test_utils/subgraph_builders/conv_pool_relu.hpp" +#include "intel_npu/al/config/common.hpp" + +#include + +#include + +using namespace ov::test::behavior; + +namespace { + +std::vector> exe_network_supported_properties = { + {ov::hint::num_requests.name(), ov::Any(8)}, + {ov::hint::enable_cpu_pinning.name(), ov::Any(true)}, + {ov::hint::performance_mode.name(), ov::Any(ov::hint::PerformanceMode::THROUGHPUT)}, + {ov::enable_profiling.name(), ov::Any(true)}, + {ov::device::id.name(), ov::Any(ov::test::utils::getDeviceNameID(ov::test::utils::getDeviceName()))}, + {ov::optimal_number_of_infer_requests.name(), ov::Any(2)}, +}; + +std::vector> exe_network_immutable_properties = { + {std::make_pair(ov::optimal_number_of_infer_requests.name(), ov::Any(2))}, + {std::make_pair(ov::hint::enable_cpu_pinning.name(), ov::Any(false))}, + {std::make_pair(ov::supported_properties.name(), ov::Any("deadbeef"))}, + {std::make_pair(ov::model_name.name(), ov::Any("deadbeef"))}}; + +std::vector> properties = {{}}; + +// ExecutableNetwork Properties tests +class ClassExecutableNetworkGetPropertiesTestNPU : + public OVCompiledModelPropertiesBase, + public ::testing::WithParamInterface< + std::tuple, std::pair>> { +protected: + std::string deviceName; + std::string configKey; + ov::Any configValue; + std::string compilerTypeConfigKey; + ov::Any compilerTypeConfigValue; + ov::Core ie; + +public: + void SetUp() override { + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + OVCompiledModelPropertiesBase::SetUp(); + deviceName = std::get<0>(GetParam()); + std::tie(configKey, configValue) = std::get<1>(GetParam()); + std::tie(compilerTypeConfigKey, compilerTypeConfigValue) = std::get<2>(GetParam()); + + model = ov::test::utils::make_conv_pool_relu(); + } + static std::string getTestCaseName( + testing::TestParamInfo< + std::tuple, std::pair>> + obj) { + std::string targetDevice; + std::pair configuration; + std::pair compilerType; + std::tie(targetDevice, configuration, compilerType) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '_'); + std::ostringstream result; + result << "targetDevice=" << ov::test::utils::getDeviceNameTestCase(targetDevice) << "_"; + result << "config=(" << configuration.first << "=" << configuration.second.as() << ")"; + result << "_compilerType=(" << compilerType.first << "=" << compilerType.second.as() << ")"; + result << "_targetPlatform=" + ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); + + return result.str(); + } +}; + +using ClassExecutableNetworkTestSuite1NPU = ClassExecutableNetworkGetPropertiesTestNPU; + +TEST_P(ClassExecutableNetworkTestSuite1NPU, PropertyIsSupportedAndImmutableAndGet) { + std::vector properties; + + ov::CompiledModel exeNetwork = + ie.compile_model(model, deviceName, {{compilerTypeConfigKey, compilerTypeConfigValue}}); + OV_ASSERT_NO_THROW(properties = exeNetwork.get_property(ov::supported_properties)); + + auto it = find(properties.cbegin(), properties.cend(), configKey); + ASSERT_TRUE(it != properties.cend()); + ASSERT_FALSE(it->is_mutable()); + + OV_ASSERT_NO_THROW(exeNetwork.get_property(configKey)); +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_ClassExecutableNetworkGetPropertiesTestNPU, + ClassExecutableNetworkTestSuite1NPU, + ::testing::Combine(::testing::Values(ov::test::utils::getDeviceName()), + ::testing::ValuesIn(exe_network_supported_properties), + ::testing::ValuesIn(properties)), + ClassExecutableNetworkTestSuite1NPU::getTestCaseName); + +using ClassExecutableNetworkTestSuite2NPU = ClassExecutableNetworkGetPropertiesTestNPU; + +TEST_P(ClassExecutableNetworkTestSuite2NPU, PropertyIsSupportedAndImmutableAndCanNotSet) { + std::vector properties; + + ov::CompiledModel exeNetwork = + ie.compile_model(model, deviceName, {{compilerTypeConfigKey, compilerTypeConfigValue}}); + OV_ASSERT_NO_THROW(properties = exeNetwork.get_property(ov::supported_properties)); + + auto it = find(properties.cbegin(), properties.cend(), configKey); + ASSERT_TRUE(it != properties.cend()); + ASSERT_FALSE(it->is_mutable()); + + ASSERT_THROW(exeNetwork.set_property({{configKey, configValue}}), ov::Exception); +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_ClassExecutableNetworkTestSuite2NPU, ClassExecutableNetworkTestSuite2NPU, + ::testing::Combine(::testing::Values(ov::test::utils::getDeviceName()), + ::testing::ValuesIn(exe_network_immutable_properties), + ::testing::ValuesIn(properties)), + ClassExecutableNetworkTestSuite2NPU::getTestCaseName); + +} // namespace + +namespace { + +std::vector> plugin_public_mutable_properties = { + {ov::hint::num_requests.name(), ov::Any(5)}, + {ov::enable_profiling.name(), ov::Any(true)}, + {ov::compilation_num_threads.name(), ov::Any(1)}, + {ov::hint::performance_mode.name(), ov::Any(ov::hint::PerformanceMode::THROUGHPUT)}, + {ov::hint::enable_cpu_pinning.name(), ov::Any(true)}, + {ov::log::level.name(), ov::Any(ov::log::Level::DEBUG)}, + {ov::device::id.name(), ov::Any(ov::test::utils::getDeviceNameID(ov::test::utils::getDeviceName()))}, +}; + +std::vector> plugin_internal_mutable_properties = { + {ov::intel_npu::compilation_mode_params.name(), ov::Any("use-user-precision=false propagate-quant-dequant=0")}, + {ov::intel_npu::dma_engines.name(), ov::Any(1)}, + {ov::intel_npu::compiler_type.name(), ov::Any(ov::intel_npu::CompilerType::MLIR)}, + {ov::intel_npu::platform.name(), ov::Any(ov::intel_npu::Platform::AUTO_DETECT)}, + {ov::intel_npu::compilation_mode.name(), ov::Any("DefaultHW")}, + {ov::intel_npu::max_tiles.name(), ov::Any(8)}, + {ov::intel_npu::stepping.name(), ov::Any(4)}, + {ov::intel_npu::dpu_groups.name(), ov::Any(2)}, + {ov::intel_npu::use_elf_compiler_backend.name(), ov::Any(ov::intel_npu::ElfCompilerBackend::YES)}, +}; + +std::vector> plugin_public_immutable_properties = { + {ov::device::uuid.name(), ov::Any("deadbeef")}, + {ov::supported_properties.name(), {ov::device::full_name.name()}}, + {ov::num_streams.name(), ov::Any(ov::streams::Num(4))}, + {ov::available_devices.name(), ov::Any(std::vector{"deadbeef"})}, + {ov::device::capabilities.name(), ov::Any(std::vector{"deadbeef"})}, + {ov::range_for_async_infer_requests.name(), + ov::Any(std::tuple{0, 10, 1})}, + {ov::range_for_streams.name(), ov::Any(std::tuple{0, 10})}, + {ov::optimal_number_of_infer_requests.name(), ov::Any(4)}, + {ov::intel_npu::device_alloc_mem_size.name(), ov::Any(2)}, + {ov::intel_npu::device_total_mem_size.name(), ov::Any(2)}, +}; + +// Plugin Properties tests +class ClassPluginPropertiesTestNPU : + public OVCompiledModelPropertiesBase, + public ::testing::WithParamInterface>> { +protected: + std::string deviceName; + std::string configKey; + ov::Any configValue; + ov::Core ie; + +public: + void SetUp() override { + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + OVCompiledModelPropertiesBase::SetUp(); + deviceName = std::get<0>(GetParam()); + std::tie(configKey, configValue) = std::get<1>(GetParam()); + } + static std::string getTestCaseName( + testing::TestParamInfo>> obj) { + std::string targetDevice; + std::pair configuration; + std::tie(targetDevice, configuration) = obj.param; + std::replace(targetDevice.begin(), targetDevice.end(), ':', '_'); + std::ostringstream result; + result << "targetDevice=" << ov::test::utils::getDeviceNameTestCase(targetDevice) << "_"; + result << "config=(" << configuration.first << "=" << configuration.second.as() << ")"; + result << "_targetPlatform=" + ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); + return result.str(); + } +}; + +using ClassPluginPropertiesTestSuite0NPU = ClassPluginPropertiesTestNPU; + +TEST_P(ClassPluginPropertiesTestSuite0NPU, CanSetGetPublicMutableProperty) { + std::vector properties; + + OV_ASSERT_NO_THROW(properties = ie.get_property(deviceName, ov::supported_properties)); + + auto it = find(properties.cbegin(), properties.cend(), configKey); + ASSERT_TRUE(it != properties.cend()); + ASSERT_TRUE(it->is_mutable()); + + OV_ASSERT_NO_THROW(ie.set_property(deviceName, {{configKey, configValue}})); + + ov::Any retrieved_value; + OV_ASSERT_NO_THROW(retrieved_value = ie.get_property(deviceName, configKey)); + + ASSERT_EQ(retrieved_value.as(), configValue.as()); +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_ClassPluginPropertiesTestNPU, ClassPluginPropertiesTestSuite0NPU, + ::testing::Combine(::testing::Values(ov::test::utils::getDeviceName()), + ::testing::ValuesIn(plugin_public_mutable_properties)), + ClassPluginPropertiesTestNPU::getTestCaseName); + +using ClassPluginPropertiesTestSuite1NPU = ClassPluginPropertiesTestNPU; + +TEST_P(ClassPluginPropertiesTestSuite1NPU, CanSetGetInternalMutableProperty) { + OV_ASSERT_NO_THROW(ie.set_property(deviceName, {{configKey, configValue}})); + + ov::Any retrieved_value; + OV_ASSERT_NO_THROW(retrieved_value = ie.get_property(deviceName, configKey)); + + ASSERT_EQ(retrieved_value.as(), configValue.as()); +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_ClassPluginPropertiesTestNPU, ClassPluginPropertiesTestSuite1NPU, + ::testing::Combine(::testing::Values(ov::test::utils::getDeviceName()), + ::testing::ValuesIn(plugin_internal_mutable_properties)), + ClassPluginPropertiesTestNPU::getTestCaseName); + +using ClassPluginPropertiesTestSuite2NPU = ClassPluginPropertiesTestNPU; + +TEST_P(ClassPluginPropertiesTestSuite2NPU, CanNotSetImmutableProperty) { + std::vector properties; + + OV_ASSERT_NO_THROW(properties = ie.get_property(deviceName, ov::supported_properties)); + + auto it = find(properties.cbegin(), properties.cend(), configKey); + ASSERT_TRUE(it != properties.cend()); + ASSERT_FALSE(it->is_mutable()); + + ov::Any orig_value; + OV_ASSERT_NO_THROW(orig_value = ie.get_property(deviceName, configKey)); + + ASSERT_THROW(ie.set_property(deviceName, {{configKey, configValue}}), ov::Exception); + + ov::Any after_value; + OV_ASSERT_NO_THROW(after_value = ie.get_property(deviceName, configKey)); + + ASSERT_EQ(orig_value.as(), after_value.as()); +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_ClassPluginPropertiesTest, ClassPluginPropertiesTestSuite2NPU, + ::testing::Combine(::testing::Values(ov::test::utils::getDeviceName()), + ::testing::ValuesIn(plugin_public_immutable_properties)), + ClassPluginPropertiesTestNPU::getTestCaseName); + +using ClassPluginPropertiesTestSuite3NPU = ClassPluginPropertiesTestNPU; + +TEST_P(ClassPluginPropertiesTestSuite3NPU, CanGetPropertyWithOptionsNotAffectingCore) { + std::vector properties; + + OV_ASSERT_NO_THROW(properties = ie.get_property(deviceName, ov::supported_properties)); + + auto it = find(properties.cbegin(), properties.cend(), configKey); + ASSERT_TRUE(it != properties.cend()); + + ov::Any retrieved_value; + OV_ASSERT_NO_THROW(retrieved_value = ie.get_property(deviceName, configKey)); + + ov::Any retrieved_value_with_options; + OV_ASSERT_NO_THROW(retrieved_value_with_options = ie.get_property( + deviceName, configKey, + {{ov::hint::performance_mode.name(), ov::Any(ov::hint::PerformanceMode::THROUGHPUT)}})); + + ov::Any retrieved_value2; + OV_ASSERT_NO_THROW(retrieved_value2 = ie.get_property(deviceName, configKey)); + + ASSERT_EQ(retrieved_value.as(), retrieved_value2.as()); +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_ClassPluginPropertiesOptsTest1NPU, ClassPluginPropertiesTestSuite3NPU, + ::testing::Combine(::testing::Values(ov::test::utils::getDeviceName()), + ::testing::ValuesIn(plugin_public_immutable_properties)), + ClassPluginPropertiesTestNPU::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_ClassPluginPropertiesOptsTest2NPU, ClassPluginPropertiesTestSuite3NPU, + ::testing::Combine(::testing::Values(ov::test::utils::getDeviceName()), + ::testing::ValuesIn(plugin_public_mutable_properties)), + ClassPluginPropertiesTestNPU::getTestCaseName); + +using ClassPluginPropertiesTestSuite4NPU = ClassExecutableNetworkGetPropertiesTestNPU; + +TEST_P(ClassPluginPropertiesTestSuite4NPU, CanNotSetGetInexistentProperty) { + // ie.set_property won't call plugin Engine::SetConfig due to empty string-ov::Plugin map from core_impl + // workaround to overcome this is to call first ie.get_property which calls get_plugin() from core_impl and + // populates plugin map + std::vector properties; + OV_ASSERT_NO_THROW(properties = ie.get_property(deviceName, ov::supported_properties)); + + ASSERT_THROW(ie.set_property(deviceName, {{configKey, configValue}}), ov::Exception); + + ASSERT_THROW(auto property1 = ie.get_property(deviceName, configKey), ov::Exception); + + ASSERT_THROW( + ov::CompiledModel compiled_model1 = ie.compile_model( + model, deviceName, {{configKey, configValue}, {compilerTypeConfigKey, compilerTypeConfigValue}}), + ov::Exception); + + ov::CompiledModel compiled_model2; + + OV_ASSERT_NO_THROW(compiled_model2 = + ie.compile_model(model, deviceName, {{compilerTypeConfigKey, compilerTypeConfigValue}})); + + ASSERT_THROW(compiled_model2.set_property({{configKey, configValue}}), + ov::Exception); // Expect to throw due to unimplemented method + + ASSERT_THROW(auto property2 = compiled_model2.get_property(configKey), + ov::Exception); // Expect to throw due to unsupported config +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_ClassExecutableNetworkGetPropertiesTestNPU, + ClassPluginPropertiesTestSuite4NPU, + ::testing::Combine(::testing::Values(ov::test::utils::getDeviceName()), + ::testing::ValuesIn({std::make_pair( + "THISCONFIGKEYNOTEXIST", ov::Any("THISCONFIGVALUENOTEXIST"))}), + ::testing::ValuesIn(properties)), + ClassPluginPropertiesTestSuite4NPU::getTestCaseName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/callback.hpp b/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/callback.hpp new file mode 100644 index 00000000000..aa8d3e5e4d4 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/callback.hpp @@ -0,0 +1,126 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "overload/overload_test_utils_npu.hpp" + +namespace ov { +namespace test { +namespace behavior { +using OVInferRequestCallbackTestsNPU = OVInferRequestTestsNPU; + +TEST_P(OVInferRequestCallbackTestsNPU, canCallAsyncWithCompletionCallback) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + bool is_called = false; + OV_ASSERT_NO_THROW(req.set_callback([&](std::exception_ptr exception_ptr) { + // HSD_1805940120: Wait on starting callback return HDDL_ERROR_INVAL_TASK_HANDLE + ASSERT_EQ(exception_ptr, nullptr); + is_called = true; + })); + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); + ASSERT_TRUE(is_called); +} + +TEST_P(OVInferRequestCallbackTestsNPU, syncInferDoesNotCallCompletionCallback) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + bool is_called = false; + req.set_callback([&](std::exception_ptr exception_ptr) { + ASSERT_EQ(nullptr, exception_ptr); + is_called = true; + }); + req.infer(); + ASSERT_FALSE(is_called); +} + +// test that can wait all callbacks on dtor +TEST_P(OVInferRequestCallbackTestsNPU, canStartSeveralAsyncInsideCompletionCallbackWithSafeDtor) { + const int NUM_ITER = 10; + struct TestUserData { + std::atomic numIter = {0}; + std::promise promise; + }; + TestUserData data; + + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req.set_callback([&](std::exception_ptr exception_ptr) { + if (exception_ptr) { + data.promise.set_exception(exception_ptr); + } else { + if (data.numIter.fetch_add(1) != NUM_ITER) { + req.start_async(); + } else { + data.promise.set_value(true); + } + } + })); + auto future = data.promise.get_future(); + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); + future.wait(); + auto callbackStatus = future.get(); + ASSERT_TRUE(callbackStatus); + auto dataNumIter = data.numIter - 1; + ASSERT_EQ(NUM_ITER, dataNumIter); +} + +TEST_P(OVInferRequestCallbackTestsNPU, returnGeneralErrorIfCallbackThrowException) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req.set_callback([](std::exception_ptr) { + OPENVINO_THROW("Throw"); + })); + OV_ASSERT_NO_THROW(req.start_async()); + ASSERT_THROW(req.wait(), ov::Exception); +} + +TEST_P(OVInferRequestCallbackTestsNPU, ReturnResultNotReadyFromWaitInAsyncModeForTooSmallTimeout) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + std::promise callbackTimeStamp; + auto callbackTimeStampFuture = callbackTimeStamp.get_future(); + // add a callback to the request and capture the timestamp + OV_ASSERT_NO_THROW(req.set_callback([&](std::exception_ptr exception_ptr) { + if (exception_ptr) { + callbackTimeStamp.set_exception(exception_ptr); + } else { + callbackTimeStamp.set_value(std::chrono::system_clock::now()); + } + })); + OV_ASSERT_NO_THROW(req.start_async()); + bool ready = false; + OV_ASSERT_NO_THROW(ready = req.wait_for({})); + // get timestamp taken AFTER return from the wait(STATUS_ONLY) + const auto afterWaitTimeStamp = std::chrono::system_clock::now(); + // IF the callback timestamp is larger than the afterWaitTimeStamp + // then we should observe false ready result + if (afterWaitTimeStamp < callbackTimeStampFuture.get()) { + ASSERT_FALSE(ready); + } + OV_ASSERT_NO_THROW(req.wait()); +} + +TEST_P(OVInferRequestCallbackTestsNPU, ImplDoesNotCopyCallback) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + { + auto somePtr = std::make_shared(42); + OV_ASSERT_NO_THROW(req.set_callback([somePtr](std::exception_ptr exception_ptr) { + ASSERT_EQ(nullptr, exception_ptr); + ASSERT_EQ(1, somePtr.use_count()); + })); + } + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/cancellation.hpp b/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/cancellation.hpp new file mode 100644 index 00000000000..711463798ea --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/cancellation.hpp @@ -0,0 +1,67 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "openvino/runtime/exception.hpp" + +#include "overload/overload_test_utils_npu.hpp" + +namespace ov { +namespace test { +namespace behavior { +using OVInferRequestCancellationTestsNPU = OVInferRequestTestsNPU; + +TEST_P(OVInferRequestCancellationTestsNPU, canCancelAsyncRequest) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.cancel()); + try { + req.wait(); + } catch (const ov::Cancelled&) { + SUCCEED(); + } +} + +TEST_P(OVInferRequestCancellationTestsNPU, CanResetAfterCancelAsyncRequest) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.cancel()); + try { + req.wait(); + } catch (const ov::Cancelled&) { + SUCCEED(); + } + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); +} + +TEST_P(OVInferRequestCancellationTestsNPU, canCancelBeforeAsyncRequest) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req.cancel()); +} + +TEST_P(OVInferRequestCancellationTestsNPU, canCancelInferRequest) { + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + auto infer = std::async(std::launch::async, [&req] { + req.infer(); + }); + while (!req.wait_for({})) { + } + OV_ASSERT_NO_THROW(req.cancel()); + try { + infer.get(); + } catch (const ov::Cancelled&) { + SUCCEED(); + } +} +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/io_tensor.hpp b/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/io_tensor.hpp new file mode 100644 index 00000000000..eb8bc5bd636 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/io_tensor.hpp @@ -0,0 +1,257 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include "behavior/ov_infer_request/io_tensor.hpp" +#include "common_test_utils/subgraph_builders/conv_pool_relu.hpp" +#include "overload/overload_test_utils_npu.hpp" + +namespace ov { +namespace test { +namespace behavior { + +struct OVInferRequestIOTensorTestNPU : public OVInferRequestIOTensorTest { + void SetUp() override { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + std::tie(target_device, configuration) = this->GetParam(); + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + APIBaseTest::SetUp(); + function = ov::test::behavior::getDefaultNGraphFunctionForTheDeviceNPU(); + ov::AnyMap params; + for (auto&& v : configuration) { + params.emplace(v.first, v.second); + } + execNet = core->compile_model(function, target_device, params); + try { + req = execNet.create_infer_request(); + } catch (const std::exception& ex) { + FAIL() << "Can't Create Infer Requiest in SetUp \nException [" << ex.what() << "]" << std::endl; + } + input = execNet.input(); + output = execNet.output(); + } +}; + +TEST_P(OVInferRequestIOTensorTestNPU, failToSetNullptrForInput) { + ASSERT_THROW(req.set_tensor(input, {}), ov::Exception); +} + +TEST_P(OVInferRequestIOTensorTestNPU, failToSetNullptrForOutput) { + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + ASSERT_THROW(req.set_tensor(output, {}), ov::Exception); +} + +TEST_P(OVInferRequestIOTensorTestNPU, canSetAndGetInput) { + auto tensor = utils::create_and_fill_tensor(input.get_element_type(), input.get_shape()); + OV_ASSERT_NO_THROW(req.set_tensor(input, tensor)); + ov::Tensor actual_tensor; + OV_ASSERT_NO_THROW(actual_tensor = req.get_tensor(input)); + + ASSERT_TRUE(actual_tensor); + ASSERT_NE(nullptr, actual_tensor.data()); + ASSERT_EQ(tensor.data(), actual_tensor.data()); + ASSERT_EQ(input.get_element_type(), actual_tensor.get_element_type()); + ASSERT_EQ(input.get_shape(), actual_tensor.get_shape()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, canSetAndGetOutput) { + auto tensor = utils::create_and_fill_tensor(output.get_element_type(), output.get_shape()); + req.set_tensor(output, tensor); + auto actual_tensor = req.get_tensor(output); + + ASSERT_TRUE(actual_tensor); + ASSERT_FALSE(actual_tensor.data() == nullptr); + ASSERT_EQ(actual_tensor.data(), tensor.data()); + ASSERT_EQ(output.get_element_type(), actual_tensor.get_element_type()); + ASSERT_EQ(output.get_shape(), actual_tensor.get_shape()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, failToSetTensorWithIncorrectName) { + auto tensor = utils::create_and_fill_tensor(input.get_element_type(), input.get_shape()); + ASSERT_THROW(req.set_tensor("incorrect_input", tensor), ov::Exception); +} + +TEST_P(OVInferRequestIOTensorTestNPU, failToSetInputWithIncorrectSizes) { + auto shape = input.get_shape(); + shape[0] *= 2; + auto tensor = utils::create_and_fill_tensor(input.get_element_type(), shape); + ASSERT_THROW(req.set_tensor(input, tensor), ov::Exception); +} + +TEST_P(OVInferRequestIOTensorTestNPU, failToSetOutputWithIncorrectSizes) { + auto shape = output.get_shape(); + shape[0] *= 2; + auto tensor = utils::create_and_fill_tensor(output.get_element_type(), shape); + ASSERT_THROW(req.set_tensor(output, tensor), ov::Exception); +} + +TEST_P(OVInferRequestIOTensorTestNPU, canInferWithoutSetAndGetInOutSync) { + OV_ASSERT_NO_THROW(req.infer()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, canInferWithoutSetAndGetInOutAsync) { + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, secondCallGetInputDoNotReAllocateData) { + ov::Tensor tensor1, tensor2; + OV_ASSERT_NO_THROW(tensor1 = req.get_tensor(input)); + OV_ASSERT_NO_THROW(tensor2 = req.get_tensor(input)); + ASSERT_EQ(tensor1.data(), tensor2.data()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, secondCallGetOutputDoNotReAllocateData) { + ov::Tensor tensor1, tensor2; + OV_ASSERT_NO_THROW(tensor1 = req.get_tensor(output)); + OV_ASSERT_NO_THROW(tensor2 = req.get_tensor(output)); + ASSERT_EQ(tensor1.data(), tensor2.data()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, secondCallGetInputAfterInferSync) { + ov::Tensor tensor1, tensor2; + OV_ASSERT_NO_THROW(req.infer()); + OV_ASSERT_NO_THROW(tensor1 = req.get_tensor(input)); + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); + OV_ASSERT_NO_THROW(tensor2 = req.get_tensor(input)); + ASSERT_EQ(tensor1.data(), tensor2.data()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, secondCallGetOutputAfterInferSync) { + ov::Tensor tensor1, tensor2; + OV_ASSERT_NO_THROW(req.infer()); + OV_ASSERT_NO_THROW(tensor1 = req.get_tensor(output)); + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); + OV_ASSERT_NO_THROW(tensor2 = req.get_tensor(output)); + ASSERT_EQ(tensor1.data(), tensor2.data()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, canInferWithSetInOutBlobs) { + auto input_tensor = utils::create_and_fill_tensor(input.get_element_type(), input.get_shape()); + OV_ASSERT_NO_THROW(req.set_tensor(input, input_tensor)); + auto output_tensor = utils::create_and_fill_tensor(output.get_element_type(), output.get_shape()); + OV_ASSERT_NO_THROW(req.set_tensor(output, output_tensor)); + OV_ASSERT_NO_THROW(req.infer()); + + auto actual_input_tensor = req.get_tensor(input); + ASSERT_EQ(actual_input_tensor.data(), input_tensor.data()); + auto actual_output_tensor = req.get_tensor(output); + ASSERT_EQ(actual_output_tensor.data(), output_tensor.data()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, canInferWithGetIn) { + ov::Tensor input_tensor; + OV_ASSERT_NO_THROW(input_tensor = req.get_tensor(input)); + OV_ASSERT_NO_THROW(req.infer()); + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); + OV_ASSERT_NO_THROW(req.get_tensor(output)); +} + +TEST_P(OVInferRequestIOTensorTestNPU, canInferAfterIOBlobReallocation) { + ov::Tensor input_tensor, output_tensor; + auto in_shape = input.get_shape(); + auto out_shape = output.get_shape(); + + // imitates blob reallocation + OV_ASSERT_NO_THROW(input_tensor = req.get_tensor(input)); + OV_ASSERT_NO_THROW(input_tensor.set_shape({5, 5, 5, 5})); + OV_ASSERT_NO_THROW(input_tensor.set_shape(in_shape)); + + OV_ASSERT_NO_THROW(output_tensor = req.get_tensor(output)); + OV_ASSERT_NO_THROW(output_tensor.set_shape({20, 20, 20, 20})); + OV_ASSERT_NO_THROW(output_tensor.set_shape(out_shape)); + + OV_ASSERT_NO_THROW(req.infer()); + OV_ASSERT_NO_THROW(req.start_async()); + OV_ASSERT_NO_THROW(req.wait()); + OV_ASSERT_NO_THROW(req.get_tensor(output)); +} + +TEST_P(OVInferRequestIOTensorTestNPU, InferStaticNetworkSetChangedInputTensorThrow) { + const ov::Shape shape1 = {1, 2, 32, 32}; + const ov::Shape shape2 = {1, 2, 40, 40}; + std::map shapes; + shapes[function->inputs().back().get_any_name()] = shape1; + OV_ASSERT_NO_THROW(function->reshape(shapes)); + // Load ov::Model to target plugins + std::shared_ptr ie = utils::PluginCache::get().core(); + auto execNet = ie->compile_model(function, target_device, configuration); + // Create InferRequest + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req.infer()); + // Get input_tensor + ov::Tensor tensor; + OV_ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); + // Set shape + OV_ASSERT_NO_THROW(tensor.set_shape(shape2)); + ASSERT_ANY_THROW(req.infer()); +} + +TEST_P(OVInferRequestIOTensorTestNPU, InferStaticNetworkSetChangedOutputTensorThrow) { + const ov::Shape shape1 = {1, 2, 32, 32}; + ov::Shape shape2; + shape2 = ov::Shape{1, 20}; + + std::map shapes; + shapes[function->inputs().back().get_any_name()] = shape1; + OV_ASSERT_NO_THROW(function->reshape(shapes)); + // Load ov::Model to target plugins + std::shared_ptr ie = utils::PluginCache::get().core(); + auto execNet = ie->compile_model(function, target_device, configuration); + // Create InferRequest + ov::InferRequest req; + OV_ASSERT_NO_THROW(req = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req.infer()); + // Get output_tensor + ov::Tensor tensor; + OV_ASSERT_NO_THROW(tensor = req.get_tensor(function->outputs().back().get_any_name());); + // Set shape + OV_ASSERT_NO_THROW(tensor.set_shape(shape2)); + ASSERT_ANY_THROW(req.infer()); +} + +struct OVInferRequestIOTensorSetPrecisionTestNPU : OVInferRequestIOTensorSetPrecisionTest { + void SetUp() override { + std::tie(element_type, target_device, config) = this->GetParam(); + SKIP_IF_CURRENT_TEST_IS_DISABLED() + APIBaseTest::SetUp(); + function = ov::test::utils::make_conv_pool_relu(); + execNet = core->compile_model(function, target_device, config); + req = execNet.create_infer_request(); + } +}; + +TEST_P(OVInferRequestIOTensorSetPrecisionTestNPU, CanSetInBlobWithDifferentPrecision) { + for (auto&& output : execNet.outputs()) { + auto output_tensor = utils::create_and_fill_tensor(element_type, output.get_shape()); + if (output.get_element_type() == element_type) { + OV_ASSERT_NO_THROW(req.set_tensor(output, output_tensor)); + } else { + ASSERT_THROW(req.set_tensor(output, output_tensor), ov::Exception); + } + } +} + +TEST_P(OVInferRequestIOTensorSetPrecisionTestNPU, CanSetOutBlobWithDifferentPrecision) { + for (auto&& input : execNet.inputs()) { + auto input_tensor = utils::create_and_fill_tensor(element_type, input.get_shape()); + if (input.get_element_type() == element_type) { + OV_ASSERT_NO_THROW(req.set_tensor(input, input_tensor)); + } else { + ASSERT_THROW(req.set_tensor(input, input_tensor), ov::Exception); + } + } +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/multithreading.hpp b/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/multithreading.hpp new file mode 100644 index 00000000000..92d77152dca --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/internal/overload/ov_infer_request/multithreading.hpp @@ -0,0 +1,102 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "overload/overload_test_utils_npu.hpp" + +namespace ov { +namespace test { +namespace behavior { +using OVInferRequestMultithreadingTestsNPU = OVInferRequestTestsNPU; + +TEST_P(OVInferRequestMultithreadingTestsNPU, canRun3SyncRequestsConsistentlyFromThreads) { + ov::InferRequest req1, req2, req3; + OV_ASSERT_NO_THROW(req1 = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req2 = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req3 = execNet.create_infer_request()); + + auto f1 = std::async(std::launch::async, [&] { + req1.infer(); + }); + auto f2 = std::async(std::launch::async, [&] { + req2.infer(); + }); + auto f3 = std::async(std::launch::async, [&] { + req3.infer(); + }); + + f1.wait(); + f2.wait(); + f3.wait(); + + OV_ASSERT_NO_THROW(f1.get()); + OV_ASSERT_NO_THROW(f2.get()); + OV_ASSERT_NO_THROW(f3.get()); +} + +TEST_P(OVInferRequestMultithreadingTestsNPU, canRun3AsyncRequestsConsistentlyFromThreadsWithoutWait) { + ov::InferRequest req1, req2, req3; + OV_ASSERT_NO_THROW(req1 = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req2 = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req3 = execNet.create_infer_request()); + + OV_ASSERT_NO_THROW(req1.infer()); + OV_ASSERT_NO_THROW(req2.infer()); + OV_ASSERT_NO_THROW(req3.infer()); + + auto f1 = std::async(std::launch::async, [&] { + req1.start_async(); + }); + auto f2 = std::async(std::launch::async, [&] { + req2.start_async(); + }); + auto f3 = std::async(std::launch::async, [&] { + req3.start_async(); + }); + + f1.wait(); + f2.wait(); + f3.wait(); + + OV_ASSERT_NO_THROW(f1.get()); + OV_ASSERT_NO_THROW(f2.get()); + OV_ASSERT_NO_THROW(f3.get()); +} + +TEST_P(OVInferRequestMultithreadingTestsNPU, canRun3AsyncRequestsConsistentlyWithWait) { + ov::InferRequest req1, req2, req3; + OV_ASSERT_NO_THROW(req1 = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req2 = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req3 = execNet.create_infer_request()); + + req1.start_async(); + OV_ASSERT_NO_THROW(req1.wait()); + + req2.start_async(); + OV_ASSERT_NO_THROW(req2.wait()); + + req3.start_async(); + OV_ASSERT_NO_THROW(req3.wait()); +} + +TEST_P(OVInferRequestMultithreadingTestsNPU, canRun3AsyncRequestsParallelWithWait) { + ov::InferRequest req1, req2, req3; + OV_ASSERT_NO_THROW(req1 = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req2 = execNet.create_infer_request()); + OV_ASSERT_NO_THROW(req3 = execNet.create_infer_request()); + + req1.start_async(); + req2.start_async(); + req3.start_async(); + + OV_ASSERT_NO_THROW(req2.wait()); + OV_ASSERT_NO_THROW(req1.wait()); + OV_ASSERT_NO_THROW(req3.wait()); +} +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/core_integration.hpp b/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/core_integration.hpp new file mode 100644 index 00000000000..2651be8f280 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/core_integration.hpp @@ -0,0 +1,223 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include "base/ov_behavior_test_utils.hpp" +#include "behavior/ov_plugin/properties_tests.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "common_test_utils/subgraph_builders/concat_with_params.hpp" +#include "common_test_utils/subgraph_builders/kso_func.hpp" +#include "common_test_utils/subgraph_builders/single_concat_with_constant.hpp" +#include "common_test_utils/subgraph_builders/split_conv_concat.hpp" +#include "intel_npu/al/config/common.hpp" + +using CompilationParams = std::tuple; + +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT +#include +#define GTEST_COUT std::cerr << "[ ] [ INFO ] " +#include +#include +#include "openvino/pass/manager.hpp" +#endif + +namespace ov { +namespace test { +namespace behavior { + +class OVClassBaseTestPNPU : + public OVClassNetworkTest, + public testing::WithParamInterface, + public OVPluginTestBase { +protected: + ov::AnyMap configuration; + std::string deathTestStyle; + std::shared_ptr function; + +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string target_device; + ov::AnyMap configuration; + std::tie(target_device, configuration) = obj.param; + std::replace(target_device.begin(), target_device.end(), ':', '.'); + + std::ostringstream result; + result << "OVClassNetworkTestName_" << target_device; + result << "_targetDevice=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU) + << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + return result.str(); + } + + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + APIBaseTest::SetUp(); + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + // Generic network + actualNetwork = ov::test::utils::make_split_conv_concat(); + // Quite simple network + simpleNetwork = ov::test::utils::make_single_concat_with_constant(); + // Multinput to substruct network + multinputNetwork = ov::test::utils::make_concat_with_params(); + // Network with KSO + ksoNetwork = ov::test::utils::make_kso_function(); + + ov::AnyMap params; + for (auto&& v : configuration) { + params.emplace(v.first, v.second); + } + } + + void TearDown() override { + ::testing::GTEST_FLAG(death_test_style) = deathTestStyle; + if (!configuration.empty()) { + utils::PluginCache::get().reset(); + } + APIBaseTest::TearDown(); + } +}; + +class OVClassBasicTestPNPU : public OVBasicPropertiesTestsP { + +public: + void TearDown() override { + for (std::size_t testIndex = 0; testIndex < ov::test::utils::test_unicode_postfix_vector.size(); testIndex++) { + std::wstring postfix = ov::test::utils::test_unicode_postfix_vector[testIndex]; + std::wstring unicode_path = ov::test::utils::stringToWString(ov::util::get_ov_lib_path() + "/") + postfix; +#ifndef _WIN32 + removeDirFilesRecursive(ov::util::wstring_to_string(unicode_path)); +#else + removeDirFilesRecursive(unicode_path); +#endif + } + } +}; + +using OVClassNetworkTestPNPU = OVClassBaseTestPNPU; +using OVClassLoadNetworkTestNPU = OVClassBaseTestPNPU; + +TEST_P(OVClassNetworkTestPNPU, LoadNetworkActualNoThrow) { + ov::Core ie = createCoreWithTemplate(); + OV_ASSERT_NO_THROW(ie.compile_model(actualNetwork, target_device, configuration)); +} + +TEST_P(OVClassNetworkTestPNPU, LoadNetworkActualHeteroDeviceNoThrow) { + ov::Core ie = createCoreWithTemplate(); + OV_ASSERT_NO_THROW(ie.compile_model( + actualNetwork, ov::test::utils::DEVICE_HETERO + std::string(":") + target_device, configuration)); +} + +TEST_P(OVClassNetworkTestPNPU, LoadNetworkActualHeteroDevice2NoThrow) { + ov::Core ie = createCoreWithTemplate(); + + OV_ASSERT_NO_THROW(ie.compile_model(actualNetwork, ov::test::utils::DEVICE_HETERO, + ov::device::priorities(target_device), + ov::device::properties(target_device, configuration))); +} + +TEST_P(OVClassNetworkTestPNPU, LoadNetworkActualHeteroDeviceUsingDevicePropertiesNoThrow) { + ov::Core ie = createCoreWithTemplate(); + configuration.emplace(ov::enable_profiling(true)); + + OV_ASSERT_NO_THROW(ie.compile_model(actualNetwork, ov::test::utils::DEVICE_HETERO, + ov::device::priorities(target_device), + ov::device::properties(target_device, configuration))); +} + +TEST_P(OVClassLoadNetworkTestNPU, LoadNetworkHETEROWithDeviceIDNoThrow) { + ov::Core ie = createCoreWithTemplate(); + + auto supported_properties = ie.get_property(target_device, ov::supported_properties); + + if (supported_properties.end() != + std::find(std::begin(supported_properties), std::end(supported_properties), ov::device::id)) { + auto deviceIDs = ie.get_property(target_device, ov::available_devices); + if (deviceIDs.empty()) + GTEST_FAIL(); + std::string heteroDevice = ov::test::utils::DEVICE_HETERO + std::string(":") + target_device + "." + + deviceIDs[0] + "," + target_device; + OV_ASSERT_NO_THROW(ie.compile_model(actualNetwork, heteroDevice, configuration)); + } else { + GTEST_FAIL() << "Device does not support DeviceID property" << std::endl; + } +} + +TEST(OVClassBasicPropsTestNPU, smoke_SetConfigDevicePropertiesThrows) { + ov::Core core; + ASSERT_THROW(core.set_property("", ov::device::properties(ov::test::utils::DEVICE_NPU, ov::enable_profiling(true))), + ov::Exception); + ASSERT_THROW(core.set_property(ov::test::utils::DEVICE_NPU, + ov::device::properties(ov::test::utils::DEVICE_NPU, ov::enable_profiling(true))), + ov::Exception); + ASSERT_THROW(core.set_property(ov::test::utils::DEVICE_AUTO, + ov::device::properties(ov::test::utils::DEVICE_NPU, ov::enable_profiling(true))), + ov::Exception); + ASSERT_THROW(core.set_property(ov::test::utils::DEVICE_AUTO, + ov::device::properties(ov::test::utils::DEVICE_NPU, ov::num_streams(4))), + ov::Exception); +} + +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT + +TEST_P(OVClassBasicTestPNPU, smoke_registerPluginsLibrariesUnicodePath) { + ov::Core core = createCoreWithTemplate(); + + const std::vector libs = {pluginName}; + + for (std::size_t testIndex = 0; testIndex < ov::test::utils::test_unicode_postfix_vector.size(); testIndex++) { + GTEST_COUT << testIndex; + std::string unicode_target_device = target_device + "_UNICODE_" + std::to_string(testIndex); + std::wstring postfix = ov::test::utils::test_unicode_postfix_vector[testIndex]; + std::wstring unicode_path = + ov::test::utils::stringToWString(ov::test::utils::getOpenvinoLibDirectory() + "/") + postfix; + try { +#ifndef _WIN32 + std::filesystem::create_directory(ov::util::wstring_to_string(unicode_path)); +#else + std::filesystem::create_directory(unicode_path); +#endif + std::string pluginNamePath = + ov::util::make_plugin_library_name(ov::util::wstring_to_string(unicode_path), pluginName); + + for (auto&& lib : libs) { + auto&& libPath = ov::test::utils::stringToWString( + ov::util::make_plugin_library_name(ov::test::utils::getOpenvinoLibDirectory(), lib)); + auto&& libPathNew = ov::test::utils::stringToWString( + ov::util::make_plugin_library_name(::ov::util::wstring_to_string(unicode_path), lib)); + bool is_copy_successfully = ov::test::utils::copyFile(libPath, libPathNew); + if (!is_copy_successfully) { + FAIL() << "Unable to copy from '" << libPath << "' to '" << libPathNew << "'"; + } + } + + OV_ASSERT_NO_THROW(core.register_plugin(pluginNamePath, unicode_target_device)); + OV_ASSERT_NO_THROW(core.get_versions(unicode_target_device)); + auto devices = core.get_available_devices(); + if (std::find_if(devices.begin(), devices.end(), [&unicode_target_device](std::string device) { + return device.find(unicode_target_device) != std::string::npos; + }) == devices.end()) { + FAIL() << unicode_target_device << " was not found within registered plugins."; + } + core.unload_plugin(unicode_target_device); + } catch (const ov::Exception& e_next) { + FAIL() << e_next.what(); + } + } +} +#endif + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/life_time.hpp b/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/life_time.hpp new file mode 100644 index 00000000000..271ac651e10 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/life_time.hpp @@ -0,0 +1,226 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "behavior/ov_plugin/life_time.hpp" +#include "common_test_utils/subgraph_builders/conv_pool_relu.hpp" + +using CompilationParams = std::tuple; + +namespace ov { +namespace test { +namespace behavior { + +class OVHoldersTestNPU : public OVPluginTestBase, public testing::WithParamInterface { +protected: + ov::AnyMap configuration; + std::string deathTestStyle; + std::shared_ptr function; + +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string target_device; + ov::AnyMap configuration; + std::tie(target_device, configuration) = obj.param; + std::replace(target_device.begin(), target_device.end(), ':', '.'); + + std::ostringstream result; + result << "target_device=" << target_device << "_"; + result << "targetPlatform=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU) + << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + return result.str(); + } + + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + APIBaseTest::SetUp(); + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + deathTestStyle = ::testing::GTEST_FLAG(death_test_style); + if (deathTestStyle == "fast") { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + } + function = ov::test::utils::make_conv_pool_relu(); + } + + void TearDown() override { + ::testing::GTEST_FLAG(death_test_style) = deathTestStyle; + if (!configuration.empty()) { + utils::PluginCache::get().reset(); + } + APIBaseTest::TearDown(); + } +}; + +#define EXPECT_NO_CRASH(_statement) EXPECT_EXIT(_statement; exit(0), testing::ExitedWithCode(0), "") + +static void release_order_test(std::vector order, const std::string& deviceName, + std::shared_ptr function, ov::AnyMap configuration) { + ov::AnyVector objects; + { + ov::Core core = createCoreWithTemplate(); + auto compiled_model = core.compile_model(function, deviceName, configuration); + auto request = compiled_model.create_infer_request(); + + objects = {core, compiled_model, request}; + } + for (auto&& i : order) { + objects.at(i) = {}; + } +} + +#ifndef __EMSCRIPTEN__ + +TEST_P(OVHoldersTestNPU, Orders) { + std::vector objects{"core", "compiled_model", "request"}; + std::vector order(objects.size()); + std::iota(order.begin(), order.end(), 0); + do { + std::stringstream order_str; + for (auto&& i : order) { + order_str << objects.at(i) << " "; + } + EXPECT_NO_CRASH(release_order_test(order, target_device, function, configuration)) + << "for order: " << order_str.str(); + } while (std::next_permutation(order.begin(), order.end())); +} + +#endif // __EMSCRIPTEN__ + +TEST_P(OVHoldersTestNPU, LoadedState) { + std::vector states; + { + ov::Core core = createCoreWithTemplate(); + auto compiled_model = core.compile_model(function, target_device, configuration); + auto request = compiled_model.create_infer_request(); + try { + states = request.query_state(); + } catch (...) { + } + } +} + +TEST_P(OVHoldersTestNPU, LoadedInferRequest) { + ov::InferRequest inferRequest; + { + ov::Core core = createCoreWithTemplate(); + auto compiled_model = core.compile_model(function, target_device, configuration); + inferRequest = compiled_model.create_infer_request(); + } +} + +TEST_P(OVHoldersTestNPU, LoadedTensor) { + ov::Tensor tensor; + { + ov::Core core = createCoreWithTemplate(); + auto compiled_model = core.compile_model(function, target_device, configuration); + auto request = compiled_model.create_infer_request(); + tensor = request.get_input_tensor(); + } +} + +TEST_P(OVHoldersTestNPU, LoadedAny) { + ov::Any any; + { + ov::Core core = createCoreWithTemplate(); + auto compiled_model = core.compile_model(function, target_device, configuration); + any = compiled_model.get_property(ov::supported_properties.name()); + } +} + +TEST_P(OVHoldersTestNPU, LoadedRemoteContext) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::RemoteContext ctx; + { + ov::Core core = createCoreWithTemplate(); + auto compiled_model = core.compile_model(function, target_device, configuration); + try { + ctx = compiled_model.get_context(); + } catch (...) { + } + } +} + +class OVHoldersTestOnImportedNetworkNPU : + public OVPluginTestBase, + public testing::WithParamInterface { +protected: + ov::AnyMap configuration; + std::string deathTestStyle; + std::shared_ptr function; + +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string target_device; + ov::AnyMap configuration; + std::tie(target_device, configuration) = obj.param; + std::replace(target_device.begin(), target_device.end(), ':', '.'); + + std::ostringstream result; + result << "target_device=" << target_device << "_"; + result << "targetPlatform=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU) + << "_"; + if (!configuration.empty()) { + for (auto& configItem : configuration) { + result << "configItem=" << configItem.first << "_"; + configItem.second.print(result); + } + } + return result.str(); + } + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + APIBaseTest::SetUp(); + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + deathTestStyle = ::testing::GTEST_FLAG(death_test_style); + if (deathTestStyle == "fast") { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + } + function = ov::test::utils::make_conv_pool_relu(); + } + void TearDown() override { + ::testing::GTEST_FLAG(death_test_style) = deathTestStyle; + if (!configuration.empty()) { + utils::PluginCache::get().reset(); + } + APIBaseTest::TearDown(); + } +}; + +TEST_P(OVHoldersTestOnImportedNetworkNPU, LoadedTensor) { + ov::Core core = createCoreWithTemplate(); + std::stringstream stream; + { + auto compiled_model = core.compile_model(function, target_device, configuration); + compiled_model.export_model(stream); + } + auto compiled_model = core.import_model(stream, target_device, configuration); + auto request = compiled_model.create_infer_request(); + ov::Tensor tensor = request.get_input_tensor(); +} + +TEST_P(OVHoldersTestOnImportedNetworkNPU, CreateRequestWithCoreRemoved) { + ov::Core core = createCoreWithTemplate(); + std::stringstream stream; + { + auto compiled_model = core.compile_model(function, target_device, configuration); + compiled_model.export_model(stream); + } + auto compiled_model = core.import_model(stream, target_device, configuration); + core = ov::Core{}; + auto request = compiled_model.create_infer_request(); +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/overload_test_utils_npu.hpp b/src/plugins/intel_npu/tests/functional/internal/overload/overload_test_utils_npu.hpp new file mode 100644 index 00000000000..5173babfffc --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/internal/overload/overload_test_utils_npu.hpp @@ -0,0 +1,35 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "base/ov_behavior_test_utils.hpp" +#include "common_test_utils/subgraph_builders/conv_pool_relu.hpp" + +namespace ov { +namespace test { +namespace behavior { + +inline std::shared_ptr getDefaultNGraphFunctionForTheDeviceNPU( + std::vector inputShape = {1, 2, 32, 32}, ov::element::Type_t ngPrc = ov::element::Type_t::f32) { + return ov::test::utils::make_conv_pool_relu(inputShape, ngPrc); +} + +class OVInferRequestTestsNPU : public OVInferRequestTests { +public: + void SetUp() override { + std::tie(target_device, configuration) = this->GetParam(); + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + APIBaseTest::SetUp(); + function = ov::test::behavior::getDefaultNGraphFunctionForTheDeviceNPU(); + ov::AnyMap params; + for (auto&& v : configuration) { + params.emplace(v.first, v.second); + } + execNet = core->compile_model(function, target_device, params); + } +}; + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/main.cpp b/src/plugins/intel_npu/tests/functional/main.cpp new file mode 100644 index 00000000000..10ad66b5b7f --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/main.cpp @@ -0,0 +1,85 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include +#include +#include "gtest/gtest.h" +#include "intel_npu/al/config/config.hpp" +#include "npu_private_properties.hpp" +#include "npu_test_report.hpp" +#include "npu_test_tool.hpp" + +namespace testing { +namespace internal { +extern bool g_help_flag; +} // namespace internal +} // namespace testing + +void sigsegv_handler(int errCode); + +void sigsegv_handler(int errCode) { + auto& s = ov::test::utils::OpSummary::getInstance(); + s.saveReport(); + std::cerr << "Unexpected application crash with code: " << errCode << std::endl; + std::abort(); +} + +int main(int argc, char** argv, char** envp) { + // register crashHandler for SIGSEGV signal + signal(SIGSEGV, sigsegv_handler); + + std::ostringstream oss; + oss << "Command line args (" << argc << "): "; + for (int c = 0; c < argc; ++c) { + oss << " " << argv[c]; + } + oss << std::endl; + + oss << "Process id: " << getpid() << std::endl; + std::cout << oss.str(); + oss.str(""); + + oss << "Environment variables: "; + for (char** env = envp; *env != 0; env++) { + oss << *env << "; "; + } + + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new ov::test::utils::NpuTestReportEnvironment()); + + const bool dryRun = ::testing::GTEST_FLAG(list_tests) || ::testing::internal::g_help_flag; + + if (!dryRun) { + const std::string noFetch{""}; + std::string backend{noFetch}, arch{noFetch}, full{noFetch}; + try { + ov::test::utils::NpuTestTool npuTestTool(ov::test::utils::NpuTestEnvConfig::getInstance()); + backend = npuTestTool.getDeviceMetric(ov::intel_npu::backend_name.name()); + arch = npuTestTool.getDeviceMetric(ov::device::architecture.name()); + full = npuTestTool.getDeviceMetric(ov::device::full_name.name()); + } catch (const std::exception& e) { + std::cerr << "Exception while trying to determine device characteristics: " << e.what() << std::endl; + } + std::cout << "Tests run with: Backend name: '" << backend << "'; Device arch: '" << arch + << "'; Full device name: '" << full << "'" << std::endl; + } + + std::string dTest = ::testing::internal::GTEST_FLAG(internal_run_death_test); + if (dTest.empty()) { + std::cout << oss.str() << std::endl; + } else { + std::cout << "gtest death test process is running" << std::endl; + } + + auto& log = intel_npu::Logger::global(); + auto level = ov::test::utils::NpuTestEnvConfig::getInstance().IE_NPU_TESTS_LOG_LEVEL; + ov::log::Level logLevel = level.empty() + ? ov::log::Level::ERR + : intel_npu::OptionParser::parse(level.c_str()); + log.setLevel(logLevel); + + return RUN_ALL_TESTS(); +} diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/compiled_model_base.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/compiled_model_base.cpp new file mode 100644 index 00000000000..3cd57626cf3 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/compiled_model_base.cpp @@ -0,0 +1,69 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/compiled_model/compiled_model_base.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; +namespace { + +const std::vector modelTypes{ov::element::f16, ov::element::f32}; + +const std::vector compiledModelConfigs = {{}}; + +// Hetero configs + +auto heteroCompiledModelConfigs = []() -> std::vector { + std::vector heteroPluginConfigs(compiledModelConfigs.size()); + for (auto it = compiledModelConfigs.cbegin(); it != compiledModelConfigs.cend(); ++it) { + auto&& distance = it - compiledModelConfigs.cbegin(); + heteroPluginConfigs.at(distance) = {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, *it)}; + } + return heteroPluginConfigs; +}(); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelBaseTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVCompiledModelBaseTest, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(heteroCompiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelBaseTestOptional, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVCompiledModelBaseTestOptional, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(heteroCompiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVAutoExecutableNetworkTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVAutoExecutableNetworkTest, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(heteroCompiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, CompiledModelSetType, + ::testing::Combine(::testing::ValuesIn(modelTypes), + ::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(heteroCompiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/import_export.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/import_export.cpp new file mode 100644 index 00000000000..8ef48613c24 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/import_export.cpp @@ -0,0 +1,89 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/compiled_model/import_export.hpp" + +#include +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; +namespace { +const std::vector modelTypes = { + ov::element::f16, + ov::element::f32, +}; + +const std::vector modelShapes = {{1, 2, 5, 5}}; + +const std::vector compiledModelConfigs = {{}}; + +std::vector convertModelTypes = []() -> std::vector { + std::vector convertedModelTypes; + for (auto&& modelType : modelTypes) { + convertedModelTypes.push_back(modelType); + } + return convertedModelTypes; +}(); + +auto heteroCompiledModelConfigs = []() -> std::vector { + std::vector heteroPluginConfigs(compiledModelConfigs.size()); + for (auto it = compiledModelConfigs.cbegin(); it != compiledModelConfigs.cend(); ++it) { + auto&& distance = it - compiledModelConfigs.cbegin(); + heteroPluginConfigs.at(distance) = {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, *it)}; + } + return heteroPluginConfigs; +}(); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledGraphImportExportTest, + ::testing::Combine(::testing::ValuesIn(convertModelTypes), + ::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVCompiledGraphImportExportTest, + ::testing::Combine(::testing::ValuesIn(convertModelTypes), + ::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(heteroCompiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelImportExportTestP, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompiledModelImportExportTestP, + ::testing::Values(std::string(ov::test::utils::DEVICE_NPU), + "HETERO:" + std::string(ov::test::utils::DEVICE_NPU)), + ov::test::utils::appendPlatformTypeTestName); + +#if defined(ENABLE_INTEL_CPU) && ENABLE_INTEL_CPU + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_HETERO_CPU, OVClassCompiledModelImportExportTestP, + ::testing::Values("HETERO:" + std::string(ov::test::utils::DEVICE_NPU) + ",CPU")); +#endif + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelGraphUniqueNodeNamesTest, + ::testing::Combine(::testing::ValuesIn(modelTypes), ::testing::ValuesIn(modelShapes), + ::testing::Values(ov::test::utils::DEVICE_NPU)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVCompiledModelGraphUniqueNodeNamesTest, + ::testing::Combine(::testing::ValuesIn(modelTypes), ::testing::ValuesIn(modelShapes), + ::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVExecGraphSerializationTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVExecGraphSerializationTest, + ::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties.cpp new file mode 100644 index 00000000000..caa4d3dccf5 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties.cpp @@ -0,0 +1,275 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/compiled_model/properties.hpp" +#include "common/functions.h" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "npu_private_properties.hpp" + +using namespace ov::test::behavior; + +namespace { + +std::vector operator+(std::vector origAnyMapVector, + const std::vector>& newPair) { + std::vector newAnyMapVector(origAnyMapVector.size() * newPair.size()); + size_t index = 0; + for (const auto& pair : newPair) { + for (auto&& anyMap : origAnyMapVector) { + ov::AnyMap newAnyMap = anyMap; + ov::AnyMap& newAnyMapRef = (anyMap.find(ov::device::properties.name()) != anyMap.end()) + ? newAnyMap.find(ov::device::properties.name()) + ->second.as() + .begin() + ->second.as() + : newAnyMap; + + if (newAnyMapRef.find(pair.first) == newAnyMapRef.end()) { + newAnyMapRef.emplace(pair); + newAnyMapVector.at(index) = newAnyMap; + ++index; + } + } + } + newAnyMapVector.resize(index); + return newAnyMapVector; +} + +const std::vector> compiledModelProperties = { + {ov::enable_profiling.name(), ov::Any(true)}, + {ov::hint::performance_mode.name(), ov::Any(ov::hint::PerformanceMode::THROUGHPUT)}, + {ov::hint::num_requests.name(), ov::Any(4u)}, + {ov::hint::inference_precision.name(), ov::Any(ov::element::i8)}, + {ov::hint::enable_cpu_pinning.name(), ov::Any(true)}, + {ov::hint::model_priority.name(), ov::Any(ov::hint::Priority::HIGH)}, + {ov::intel_npu::tiles.name(), ov::Any(2)}, + {ov::intel_npu::profiling_type.name(), ov::Any(ov::intel_npu::ProfilingType::INFER)}, + {ov::intel_npu::use_elf_compiler_backend.name(), ov::Any(ov::intel_npu::ElfCompilerBackend::NO)}, + {ov::intel_npu::create_executor.name(), ov::Any(2)}}; + +const std::vector publicCompiledModelConfigs = { + {{ov::device::id.name(), ov::Any("")}}, + {{ov::hint::enable_cpu_pinning.name(), ov::Any(false)}}, + {{ov::hint::model_priority.name(), ov::Any(ov::hint::Priority::MEDIUM)}}, + {{ov::execution_devices.name(), ov::Any(ov::test::utils::DEVICE_NPU)}}, + {{ov::hint::execution_mode.name(), ov::Any(ov::hint::ExecutionMode::PERFORMANCE)}}, + {{ov::hint::inference_precision.name(), ov::Any(ov::element::f16)}}, + {{ov::loaded_from_cache.name(), ov::Any(false)}}, + {{ov::model_name.name(), ov::Any("")}}, + {{ov::optimal_number_of_infer_requests.name(), ov::Any(1u)}}, + {{ov::hint::performance_mode.name(), ov::Any(ov::hint::PerformanceMode::LATENCY)}}, + {{ov::hint::num_requests.name(), ov::Any(1u)}}, + {{ov::enable_profiling.name(), ov::Any(false)}}, + {{ov::supported_properties.name(), // needed for HETERO + ov::Any(std::vector{ + ov::PropertyName(ov::device::id.name()), ov::PropertyName(ov::hint::enable_cpu_pinning.name()), + ov::PropertyName(ov::execution_devices.name()), ov::PropertyName(ov::hint::execution_mode.name()), + ov::PropertyName(ov::hint::inference_precision.name()), + ov::PropertyName(ov::loaded_from_cache.name()), ov::PropertyName(ov::hint::model_priority.name()), + ov::PropertyName(ov::model_name.name()), + ov::PropertyName(ov::optimal_number_of_infer_requests.name()), + ov::PropertyName(ov::hint::performance_mode.name()), ov::PropertyName(ov::hint::num_requests.name()), + ov::PropertyName(ov::enable_profiling.name()), ov::PropertyName(ov::supported_properties.name())})}}}; + +const std::vector compiledModelIncorrectConfigs = { + {{"NPU_INEXISTENT_PROPERTY", "NPU_INEXISTENT_PROPERTY_VALUE"}}}; + +const std::vector> allModelPriorities = { + ov::hint::model_priority(ov::hint::Priority::LOW), ov::hint::model_priority(ov::hint::Priority::MEDIUM), + ov::hint::model_priority(ov::hint::Priority::HIGH)}; + +std::vector> compiledModelPropertiesAnyToString = + []() -> const std::vector> { + std::vector> compiledModelProps(compiledModelProperties.size()); + for (auto it = compiledModelProperties.cbegin(); it != compiledModelProperties.cend(); ++it) { + auto&& distance = it - compiledModelProperties.cbegin(); + compiledModelProps.at(distance) = {it->first, it->second.as()}; + } + return compiledModelProps; +}(); + +std::vector compiledModelConfigs = []() -> std::vector { + std::vector compiledModelConfigsMap(compiledModelProperties.size()); + for (auto it = compiledModelProperties.cbegin(); it != compiledModelProperties.cend(); ++it) { + auto&& distance = it - compiledModelProperties.cbegin(); + compiledModelConfigsMap.at(distance) = {*it}; + } + return compiledModelConfigsMap; +}(); + +auto heteroCompiledModelConfigs = []() -> std::vector { + std::vector heteroConfigs(compiledModelConfigs.size()); + for (auto it = compiledModelConfigs.cbegin(); it != compiledModelConfigs.cend(); ++it) { + auto&& distance = it - compiledModelConfigs.cbegin(); + heteroConfigs.at(distance) = { + ov::device::priorities(ov::test::utils::DEVICE_NPU), + {ov::device::properties.name(), ov::Any(ov::AnyMap{{ov::test::utils::DEVICE_NPU, ov::Any(*it)}})}}; + } + return heteroConfigs; +}(); + +auto heteroCompiledModelPublicConfigs = []() -> ov::AnyMap { + ov::AnyMap heteroPublicConfigs = { + ov::device::priorities(ov::test::utils::DEVICE_NPU), + {ov::device::properties.name(), ov::Any(ov::AnyMap{{ov::test::utils::DEVICE_NPU, ov::Any(ov::AnyMap{})}})}}; + auto& devicePropertiesAnyMap = heteroPublicConfigs.find(ov::device::properties.name()) + ->second.as() + .begin() + ->second.as(); + for (auto&& publicCompiledModelProp : publicCompiledModelConfigs) { + devicePropertiesAnyMap.emplace(*publicCompiledModelProp.begin()); + } + return heteroPublicConfigs; +}(); + +auto combineParamsExecDevices = []() -> std::vector> { + std::vector> execParams(compiledModelConfigs.size()); + for (auto it = compiledModelConfigs.cbegin(); it != compiledModelConfigs.cend(); ++it) { + auto&& distance = it - compiledModelConfigs.cbegin(); + execParams.at(distance) = std::make_pair(*it, ov::test::utils::DEVICE_NPU); + } + return execParams; +}(); + +auto combineHeteroParamsExecDevices = []() -> std::vector> { + std::vector> execHeteroParams(heteroCompiledModelConfigs.size()); + for (auto it = heteroCompiledModelConfigs.cbegin(); it != heteroCompiledModelConfigs.cend(); ++it) { + auto&& distance = it - heteroCompiledModelConfigs.cbegin(); + execHeteroParams.at(distance) = std::make_pair(*it, ov::test::utils::DEVICE_NPU); + } + return execHeteroParams; +}(); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelGetIncorrectPropertyTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompiledModelGetIncorrectPropertyTest, + ::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelGetPropertyTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompiledModelGetPropertyTest, + ::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelGetConfigTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelPropertiesTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompiledModelPropertiesTests, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(heteroCompiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompileModelWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompileModelWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(heteroCompiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelPropertiesIncorrectTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelIncorrectConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelIncorrectDevice, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelPropertiesDefaultSupportedTests, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelPropertiesDefaultTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(publicCompiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompiledModelPropertiesDefaultTests, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::Values(heteroCompiledModelPublicConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelSetCorrectConfigTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelPropertiesAnyToString)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompiledModelSetCorrectConfigTest, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelPropertiesAnyToString)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelSetIncorrectConfigTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompiledModelSetIncorrectConfigTest, + ::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests, OVClassCompiledModelGetPropertyTest_MODEL_PRIORITY, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(compiledModelConfigs + allModelPriorities)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Hetero_BehaviorTests, OVClassCompiledModelGetPropertyTest_MODEL_PRIORITY, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(heteroCompiledModelConfigs + allModelPriorities)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P( + smoke_Hetero_BehaviorTests, OVClassCompiledModelGetPropertyTest_DEVICE_PRIORITY, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(heteroCompiledModelConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVClassCompiledModelGetPropertyTest_EXEC_DEVICES, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(combineParamsExecDevices)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompiledModelGetPropertyTest_EXEC_DEVICES, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(combineHeteroParamsExecDevices)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompileModelGetExecutionDeviceTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(combineParamsExecDevices)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVCompileModelGetExecutionDeviceTests, + ::testing::Combine(::testing::Values(std::string(ov::test::utils::DEVICE_HETERO) + ":" + + ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(combineHeteroParamsExecDevices)), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties_hetero.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties_hetero.cpp new file mode 100644 index 00000000000..6aac3cb9ca4 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties_hetero.cpp @@ -0,0 +1,30 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/compiled_model/properties_hetero.hpp" +#include "common/functions.h" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "functional_test_utils/ov_plugin_cache.hpp" + +using namespace ov::test::behavior; + +namespace { + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests, OVClassHeteroCompiledModelGetMetricTest_SUPPORTED_CONFIG_KEYS, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests, OVClassHeteroCompiledModelGetMetricTest_TARGET_FALLBACK, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests, OVClassHeteroCompiledModelGetMetricTest_EXEC_DEVICES, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/batched_tensors.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/batched_tensors.cpp new file mode 100644 index 00000000000..212be818ab2 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/batched_tensors.cpp @@ -0,0 +1,17 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_infer_request/batched_tensors.hpp" +#include +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" + +using namespace ov::test::behavior; + +namespace { + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestBatchedTests, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/callback.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/callback.cpp new file mode 100644 index 00000000000..0a3e4df26c5 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/callback.cpp @@ -0,0 +1,56 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "behavior/ov_infer_request/callback.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" +#include "overload/ov_infer_request/callback.hpp" + +using namespace ov::test::behavior; + +namespace { +const std::vector configs = {{}}; + +const std::vector multiConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, ov::AnyMap{})}}; + +const std::vector autoConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, ov::AnyMap{})}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestCallbackTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestCallbackTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestCallbackTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestCallbackTestsNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestCallbackTestsNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestCallbackTestsNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/cancellation.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/cancellation.cpp new file mode 100644 index 00000000000..9a0a91f7aa4 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/cancellation.cpp @@ -0,0 +1,26 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_infer_request/cancellation.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" +#include "overload/ov_infer_request/cancellation.hpp" + +using namespace ov::test::behavior; + +namespace { +const std::vector configs = {{}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestCancellationTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestCancellationTestsNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/compile_and_infer.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/compile_and_infer.cpp new file mode 100644 index 00000000000..c617619f6d1 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/compile_and_infer.cpp @@ -0,0 +1,22 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "overload/compile_and_infer.hpp" +#include +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" + +namespace { + +using namespace ov::test::behavior; + +const std::vector configs = {{}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompileAndInferRequest, + ::testing::Combine(::testing::Values(getConstantGraph(ov::element::f32)), + ::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/infer_consistency.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/infer_consistency.cpp new file mode 100644 index 00000000000..57f56bef2a2 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/infer_consistency.cpp @@ -0,0 +1,97 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + +#include "behavior/ov_infer_request/infer_consistency.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "npu_private_properties.hpp" + +using namespace ov::test::behavior; + +namespace { +// for deviceConfigs, the deviceConfigs[0] is target device which need to be tested. +// deviceConfigs[1], deviceConfigs[2],deviceConfigs[n] are the devices which will +// be compared with target device, the result of target should be in one of the compared +// device. +using Configs = std::vector>; + +auto configs = []() { + return std::vector{{{ov::test::utils::DEVICE_NPU, + {} + }}}; +}(); + +auto autoConfigs = []() { + return std::vector{{{ov::test::utils::DEVICE_AUTO + std::string(":") + ov::test::utils::DEVICE_NPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}}, + {ov::test::utils::DEVICE_NPU, {}}}, + {{ov::test::utils::DEVICE_AUTO + std::string(":") + ov::test::utils::DEVICE_NPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}}, + {ov::test::utils::DEVICE_NPU, {}}}, + {{ov::test::utils::DEVICE_AUTO + std::string(":") + ov::test::utils::DEVICE_NPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)}}, + {ov::test::utils::DEVICE_NPU, {}}}, + {{ov::test::utils::DEVICE_AUTO + std::string(":") + ov::test::utils::DEVICE_NPU + "," + + ov::test::utils::DEVICE_CPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}}, + {ov::test::utils::DEVICE_NPU, {}}, + {ov::test::utils::DEVICE_CPU, {}}}, + {{ov::test::utils::DEVICE_AUTO + std::string(":") + ov::test::utils::DEVICE_NPU + "," + + ov::test::utils::DEVICE_CPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}}, + {ov::test::utils::DEVICE_NPU, {}}, + {ov::test::utils::DEVICE_CPU, {}}}, + {{ov::test::utils::DEVICE_AUTO + std::string(":") + ov::test::utils::DEVICE_NPU + "," + + ov::test::utils::DEVICE_CPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)}}, + {ov::test::utils::DEVICE_NPU, {}}, + {ov::test::utils::DEVICE_CPU, {}}}, + {{ov::test::utils::DEVICE_AUTO + std::string(":") + ov::test::utils::DEVICE_CPU + "," + + ov::test::utils::DEVICE_NPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)}}, + {ov::test::utils::DEVICE_CPU, {}}, + {ov::test::utils::DEVICE_NPU, {}}}}; +}(); + +auto multiConfigs = []() { + return std::vector{{{ov::test::utils::DEVICE_MULTI + std::string(":") + ov::test::utils::DEVICE_NPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}}, + {ov::test::utils::DEVICE_NPU, {}}}, + {{ov::test::utils::DEVICE_MULTI + std::string(":") + ov::test::utils::DEVICE_NPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}}, + {ov::test::utils::DEVICE_NPU, {}}}, + {{ov::test::utils::DEVICE_MULTI + std::string(":") + ov::test::utils::DEVICE_NPU + "," + + ov::test::utils::DEVICE_CPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}}, + {ov::test::utils::DEVICE_NPU, {}}, + {ov::test::utils::DEVICE_CPU, {}}}, + {{ov::test::utils::DEVICE_MULTI + std::string(":") + ov::test::utils::DEVICE_NPU + "," + + ov::test::utils::DEVICE_CPU, + {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}}, + {ov::test::utils::DEVICE_NPU, {}}, + {ov::test::utils::DEVICE_CPU, {}}}}; +}(); + +// 3x5 configuration takes ~65 seconds to run, which is already pretty long time +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_BehaviorTests, OVInferConsistencyTest, + ::testing::Combine(::testing::Values(3), // inferRequest num + ::testing::Values(5), // infer counts + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Auto_BehaviorTests, OVInferConsistencyTest, + ::testing::Combine(::testing::Values(3), // inferRequest num + ::testing::Values(5), // infer counts + ::testing::ValuesIn(autoConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Multi_BehaviorTests, OVInferConsistencyTest, + ::testing::Combine(::testing::Values(3), // inferRequest num + ::testing::Values(5), // infer counts + ::testing::ValuesIn(multiConfigs)), + ov::test::utils::appendPlatformTypeTestName); +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/inference.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/inference.cpp new file mode 100644 index 00000000000..f7c26779ed4 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/inference.cpp @@ -0,0 +1,18 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_infer_request/inference.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" + +namespace { + +using namespace ov::test::behavior; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestInferenceTests, + ::testing::Combine(::testing::Values(tensor_roi::roi_nchw(), tensor_roi::roi_1d()), + ::testing::Values(ov::test::utils::DEVICE_NPU)), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/inference_chaining.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/inference_chaining.cpp new file mode 100644 index 00000000000..27cdb01e147 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/inference_chaining.cpp @@ -0,0 +1,26 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_infer_request/inference_chaining.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "common_test_utils/test_constants.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; + +namespace { +const std::vector configs = {{}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferenceChaining, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferenceChainingStatic, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/io_tensor.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/io_tensor.cpp new file mode 100644 index 00000000000..b3e18dc2b5c --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/io_tensor.cpp @@ -0,0 +1,102 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "behavior/ov_infer_request/io_tensor.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" +#include "overload/ov_infer_request/io_tensor.hpp" + +using namespace ov::test::behavior; + +namespace { +const std::vector configs = {{}}; + +const std::vector multiConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, {})}}; + +const std::vector autoConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, {})}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestIOTensorTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestIOTensorTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestIOTensorTestNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestIOTensorTestNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestIOTensorTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +const std::vector prcs = { + ov::element::boolean, ov::element::bf16, ov::element::f16, ov::element::f32, ov::element::f64, ov::element::i4, + ov::element::i8, ov::element::i16, ov::element::i32, ov::element::i64, ov::element::u1, ov::element::u4, + ov::element::u8, ov::element::u16, ov::element::u32, ov::element::u64, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestIOTensorSetPrecisionTest, + ::testing::Combine(::testing::ValuesIn(prcs), ::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestIOTensorSetPrecisionTest, + ::testing::Combine(::testing::ValuesIn(prcs), ::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestIOTensorSetPrecisionTest, + ::testing::Combine(::testing::ValuesIn(prcs), ::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestIOTensorSetPrecisionTestNPU, + ::testing::Combine(::testing::ValuesIn(prcs), ::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Mutli_BehaviorTests, OVInferRequestIOTensorSetPrecisionTestNPU, + ::testing::Combine(::testing::ValuesIn(prcs), ::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestIOTensorSetPrecisionTestNPU, + ::testing::Combine(::testing::ValuesIn(prcs), ::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestCheckTensorPrecision, + ::testing::Combine(::testing::ValuesIn(prcs), ::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestCheckTensorPrecision, + ::testing::Combine(::testing::ValuesIn(prcs), ::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestCheckTensorPrecision, + ::testing::Combine(::testing::ValuesIn(prcs), ::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/iteration_chaining.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/iteration_chaining.cpp new file mode 100644 index 00000000000..3e6ef6cf66a --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/iteration_chaining.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_infer_request/iteration_chaining.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "npu_private_properties.hpp" + +using namespace ov::test::behavior; + +namespace { + +const std::vector configs = { + {{ov::hint::inference_precision.name(), ov::element::f32}}}; + +const std::vector heteroConfigs = { + {{ov::hint::inference_precision.name(), ov::element::f32}, + {ov::device::priorities(ov::test::utils::DEVICE_NPU)}, + {ov::device::properties(ov::test::utils::DEVICE_NPU, + {})}}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVIterationChaining, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVIterationChaining, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_HETERO), + ::testing::ValuesIn(heteroConfigs)), + ov::test::utils::appendPlatformTypeTestName); +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/memory_states.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/memory_states.cpp new file mode 100644 index 00000000000..fa3a67ffa4c --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/memory_states.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_infer_request/memory_states.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "npu_private_properties.hpp" + +using namespace ov::test::behavior; +using namespace ov; + +namespace { +const std::vector memoryStateTestCases = {memoryStateParams( + OVInferRequestVariableStateTest::get_network(), {"c_1-3", "r_1-3"}, ov::test::utils::DEVICE_NPU, + {})}; + +const std::vector memoryStateHeteroTestCases = {memoryStateParams( + OVInferRequestVariableStateTest::get_network(), {"c_1-3", "r_1-3"}, ov::test::utils::DEVICE_HETERO, + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, + {})})}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_VariableState, OVInferRequestVariableStateTest, + ::testing::ValuesIn(memoryStateTestCases), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests_VariableState, OVInferRequestVariableStateTest, + ::testing::ValuesIn(memoryStateHeteroTestCases), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/multithreading.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/multithreading.cpp new file mode 100644 index 00000000000..ab7ca572dcf --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/multithreading.cpp @@ -0,0 +1,56 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "behavior/ov_infer_request/multithreading.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" +#include "overload/ov_infer_request/multithreading.hpp" + +using namespace ov::test::behavior; + +namespace { +const std::vector configs = {{}}; + +const std::vector multiConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, {})}}; + +const std::vector autoConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, {})}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestMultithreadingTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestMultithreadingTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestMultithreadingTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestMultithreadingTestsNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestMultithreadingTestsNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestMultithreadingTestsNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + InferRequestParamsAnyMapTestName::getTestCaseName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/perf_counters.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/perf_counters.cpp new file mode 100644 index 00000000000..243d1f79fbc --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/perf_counters.cpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_infer_request/perf_counters.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; + +namespace { +const std::vector configs = {{}}; + +const std::vector multiConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}, + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}}; + +const std::vector autoConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}, + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}}; + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestPerfCountersTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestPerfCountersTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestPerfCountersTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestPerfCountersExceptionTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestPerfCountersExceptionTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestPerfCountersExceptionTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/properties_tests.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/properties_tests.cpp new file mode 100644 index 00000000000..080d2e7940e --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/properties_tests.cpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_infer_request/properties_tests.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "npu_private_properties.hpp" + +using namespace ov::test::behavior; +namespace { + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests, InferRequestPropertiesTest, + ::testing::Combine(::testing::Values(2u), ::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(std::vector{ + {}})), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/wait.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/wait.cpp new file mode 100644 index 00000000000..5ac3b4d3391 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_infer_request/wait.cpp @@ -0,0 +1,40 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "behavior/ov_infer_request/wait.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; + +namespace { +const std::vector configs = {{}}; + +const std::vector multiConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, {})}}; + +const std::vector autoConfigs = { + {ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::device::properties(ov::test::utils::DEVICE_NPU, {})}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferRequestWaitTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVInferRequestWaitTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_MULTI), + ::testing::ValuesIn(multiConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestWaitTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_AUTO), + ::testing::ValuesIn(autoConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/caching_tests.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/caching_tests.cpp new file mode 100644 index 00000000000..eee46865560 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/caching_tests.cpp @@ -0,0 +1,127 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/caching_tests.hpp" +#include +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" + +using namespace ov::test::behavior; + +namespace { +static const std::vector nightly_precisionsNPU = { + ov::element::f32, + // ov::element::f16, + // ov::element::u8, +}; + +static const std::vector smoke_precisionsNPU = { + ov::element::f32, +}; + +static const std::vector batchSizesNPU = {1}; + +static std::vector smoke_functions() { + auto funcs = CompileModelCacheTestBase::getStandardFunctions(); + if (funcs.size() > 2) { + funcs.erase(funcs.begin() + 1, funcs.end()); + } + return funcs; +} + +static std::vector NPU_functions() { + auto funcs = CompileModelCacheTestBase::getStandardFunctions(); + + std::vector::iterator it = remove_if(funcs.begin(), funcs.end(), [](ovModelWithName func) { + std::vector bad_layers{"ReadConcatSplitAssign", "SimpleFunctionRelu", + "2InputSubtract", "MatMulBias", + "TIwithLSTMcell1", "KSOFunction"}; + return std::find(bad_layers.begin(), bad_layers.end(), std::get<1>(func)) != bad_layers.end(); + }); + + funcs.erase(it, funcs.end()); + + return funcs; +} + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_CachingSupportCase_NPU, CompileModelCacheTestBase, + ::testing::Combine(::testing::ValuesIn(smoke_functions()), + ::testing::ValuesIn(smoke_precisionsNPU), + ::testing::ValuesIn(batchSizesNPU), + ::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::Values(ov::AnyMap{})), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(nightly_BehaviorTests_CachingSupportCase_NPU, CompileModelCacheTestBase, + ::testing::Combine(::testing::ValuesIn(NPU_functions()), + ::testing::ValuesIn(nightly_precisionsNPU), + ::testing::ValuesIn(batchSizesNPU), + ::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::Values(ov::AnyMap{})), + ov::test::utils::appendPlatformTypeTestName); + +static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string testCaseName = CompileModelLoadFromFileTestBase::getTestCaseName(obj); + std::replace(testCaseName.begin(), testCaseName.end(), ':', '.'); + return testCaseName + + "_targetPlatform=" + ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); +} + +const std::vector LoadFromFileConfigs = { + {ov::device::properties(ov::test::utils::DEVICE_NPU, {}), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}, + {ov::device::properties(ov::test::utils::DEVICE_NPU, {}), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}}; + +const std::vector TestTargets = { + ov::test::utils::DEVICE_AUTO, + ov::test::utils::DEVICE_MULTI, + ov::test::utils::DEVICE_BATCH, +}; + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Auto_BehaviorTests_CachingSupportCase_NPU, + CompileModelLoadFromFileTestBase, + ::testing::Combine(::testing::ValuesIn(TestTargets), ::testing::ValuesIn(LoadFromFileConfigs)), + getTestCaseName); + +const std::vector NPULoadFromFileConfigs = { + {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}, + {ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)} +}; + +const std::vector> NPUCompiledKernelsCacheTest = { + std::make_pair( + {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::intel_npu::use_elf_compiler_backend(ov::intel_npu::ElfCompilerBackend::NO)}, + "blob"), + std::make_pair( + {ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), + ov::intel_npu::use_elf_compiler_backend(ov::intel_npu::ElfCompilerBackend::NO)}, + "blob"), + std::make_pair( + {ov::intel_npu::use_elf_compiler_backend(ov::intel_npu::ElfCompilerBackend::NO)}, + "blob"), + std::make_pair({ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}, + "blob"), + std::make_pair({ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}, + "blob") +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_CachingSupportCase_NPU, CompileModelLoadFromFileTestBase, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(NPULoadFromFileConfigs)), + getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_CachingSupportCase_NPU, CompileModelLoadFromMemoryTestBase, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(NPULoadFromFileConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_CachingSupportCase_NPU, CompiledKernelsCacheTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(NPUCompiledKernelsCacheTest)), + ov::test::utils::appendPlatformTypeTestName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp new file mode 100644 index 00000000000..390b46aa162 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp @@ -0,0 +1,304 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "overload/ov_plugin/core_integration.hpp" +#include "behavior/compiled_model/properties.hpp" +#include "behavior/ov_plugin/core_integration_sw.hpp" +#include "behavior/ov_plugin/properties_tests.hpp" +#include "common/functions.h" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "common_test_utils/subgraph_builders/conv_pool_relu.hpp" +#include "common_test_utils/test_assertions.hpp" +#include "functional_test_utils/ov_plugin_cache.hpp" +#include "intel_npu/al/config/common.hpp" +#include "openvino/runtime/intel_npu/properties.hpp" + +using namespace ov::test::behavior; + +namespace { + +const char* NPU_PLUGIN_LIB_NAME = "openvino_intel_npu_plugin"; + + +std::vector devices = { + std::string(ov::test::utils::DEVICE_NPU), +}; + +std::pair plugins[] = { + std::make_pair(std::string(NPU_PLUGIN_LIB_NAME), std::string(ov::test::utils::DEVICE_NPU)), +}; + +namespace OVClassBasicTestName { +static std::string getTestCaseName(testing::TestParamInfo> obj) { + std::ostringstream result; + result << "OVClassBasicTestName_" << obj.param.first << "_" << obj.param.second; + result << "_targetDevice=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); + + return result.str(); +} +} // namespace OVClassBasicTestName + +namespace OVClassNetworkTestName { +static std::string getTestCaseName(testing::TestParamInfo obj) { + std::ostringstream result; + result << "OVClassNetworkTestName_" << obj.param; + result << "_targetDevice=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); + + return result.str(); +} +} // namespace OVClassNetworkTestName + +// +// IE Class Common tests with +// + +const std::vector configs = {{}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVBasicPropertiesTestsP, OVBasicPropertiesTestsP, + ::testing::ValuesIn(plugins), OVClassBasicTestName::getTestCaseName); + +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassBasicTestP, OVClassBasicTestPNPU, ::testing::ValuesIn(plugins), + OVClassBasicTestName::getTestCaseName); +#endif + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassModelTestP, OVClassModelTestP, ::testing::ValuesIn(devices), + OVClassNetworkTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassNetworkTestP, OVClassNetworkTestPNPU, + ::testing::Combine(::testing::ValuesIn(devices), ::testing::ValuesIn(configs)), + OVClassNetworkTestPNPU::getTestCaseName); + +// +// IE Class GetMetric +// + +INSTANTIATE_TEST_SUITE_P(BehaviorTests_OVGetMetricPropsTest_nightly, OVGetMetricPropsTest, ::testing::ValuesIn(devices), + OVClassNetworkTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(BehaviorTests_OVGetMetricPropsTest_nightly, OVGetMetricPropsOptionalTest, + ::testing::ValuesIn(devices), OVClassNetworkTestName::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + BehaviorTests_OVCheckSetSupportedRWMandatoryMetricsPropsTests, OVCheckSetSupportedRWMetricsPropsTests, + ::testing::Combine(::testing::Values("MULTI", "AUTO"), + ::testing::ValuesIn(OVCheckSetSupportedRWMetricsPropsTests::getRWOptionalPropertiesValues( + {ov::log::level.name()}))), + ov::test::utils::appendPlatformTypeTestName); + +const std::vector multiConfigs = {{ov::device::priorities(ov::test::utils::DEVICE_NPU)}}; +const std::vector configsDeviceProperties = { + {ov::device::properties(ov::test::utils::DEVICE_NPU, ov::num_streams(4))}}; +const std::vector configsWithSecondaryProperties = { + {ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +const std::vector multiConfigsWithSecondaryProperties = { + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +const std::vector autoConfigsWithSecondaryProperties = { + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties("AUTO", ov::enable_profiling(false), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties("AUTO", ov::enable_profiling(false), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties("AUTO", ov::enable_profiling(false), + ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassSetDevicePriorityConfigPropsTest, + OVClassSetDevicePriorityConfigPropsTest, + ::testing::Combine(::testing::Values("MULTI", "AUTO"), ::testing::ValuesIn(multiConfigs)), + ov::test::utils::appendPlatformTypeTestName); + +// +// IE Class GetConfig +// + +INSTANTIATE_TEST_SUITE_P(BehaviorTests_OVGetConfigTest_nightly, OVGetConfigTest, ::testing::ValuesIn(devices), + OVClassNetworkTestName::getTestCaseName); + +// IE Class Load network + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassCompileModelWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU, "AUTO:NPU", "MULTI:NPU"), + ::testing::ValuesIn(configsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests_OVClassCompileModelWithCorrectPropertiesTest, + OVClassCompileModelWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("MULTI"), + ::testing::ValuesIn(multiConfigsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(smoke_AUTO_BehaviorTests_OVClassCompileModelWithCorrectPropertiesTest, + OVClassCompileModelWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("AUTO"), + ::testing::ValuesIn(autoConfigsWithSecondaryProperties))); + +// IE Class load and check network with ov::device::properties +// OVClassCompileModelAndCheckSecondaryPropertiesTest only works with property num_streams of type int32_t +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_BehaviorTests_OVClassLoadNetworkAndCheckWithSecondaryPropertiesTest, + OVClassCompileModelAndCheckSecondaryPropertiesTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU, "AUTO:NPU", "MULTI:NPU"), + ::testing::ValuesIn(configsDeviceProperties))); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassLoadNetworkTest, OVClassLoadNetworkTestNPU, + ::testing::Combine(::testing::ValuesIn(devices), ::testing::ValuesIn(configs)), + OVClassLoadNetworkTestNPU::getTestCaseName); + +// +// NPU specific metrics +// + +using OVClassGetMetricAndPrintNoThrow = OVClassBaseTestP; +TEST_P(OVClassGetMetricAndPrintNoThrow, DeviceAllocMemSizeLesserThanTotalMemSizeNPU) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_total_mem_size.name())); + uint64_t t = p.as(); + ASSERT_NE(t, 0); + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a = p.as(); + + ASSERT_LT(a, t); + + std::cout << "OV NPU device alloc/total memory size: " << a << "/" << t << std::endl; +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, DeviceAllocMemSizeLesserAfterModelIsLoadedNPU) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a1 = p.as(); + + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + auto model = ov::test::utils::make_conv_pool_relu(); + OV_ASSERT_NO_THROW(ie.compile_model( + model, target_device, {})); + } + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a2 = p.as(); + + std::cout << "OV NPU device {alloc before load network/alloc after load network} memory size: {" << a1 << "/" << a2 + << "}" << std::endl; + + // after the network is loaded onto device, allocated memory value should increase + ASSERT_LE(a1, a2); +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, VpuDeviceAllocMemSizeLesserAfterModelIsLoaded) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a1 = p.as(); + + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + auto model = ov::test::utils::make_conv_pool_relu(); + OV_ASSERT_NO_THROW( + ie.compile_model(model, target_device, + ov::AnyMap{ov::log::level(ov::log::Level::DEBUG)})); + } + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a2 = p.as(); + + std::cout << "OV NPU device {alloc before load network/alloc after load network} memory size: {" << a1 << "/" << a2 + << "}" << std::endl; + + // after the network is loaded onto device, allocated memory value should increase + ASSERT_LE(a1, a2); +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, DriverVersionNPU) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::driver_version.name())); + uint32_t t = p.as(); + + std::cout << "NPU driver version is " << t << std::endl; + + OV_ASSERT_PROPERTY_SUPPORTED(ov::intel_npu::driver_version.name()); +} + +using OVClassCompileModel = OVClassBaseTestP; +TEST_P(OVClassCompileModel, CompileModelWithDifferentThreadNumbers) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + auto model = ov::test::utils::make_conv_pool_relu(); + OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(1)}})); + + OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(2)}})); + + OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(4)}})); + + EXPECT_ANY_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(-1)}})); + OV_EXPECT_THROW( + std::ignore = ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(-1)}}), + ::ov::Exception, testing::HasSubstr("ov::compilation_num_threads must be positive int32 value")); +} + +INSTANTIATE_TEST_SUITE_P(nightly_BehaviorTests_OVClassGetMetricTest, OVClassGetMetricAndPrintNoThrow, + ::testing::Values(ov::test::utils::DEVICE_NPU), OVClassNetworkTestName::getTestCaseName); + +// Several devices case +INSTANTIATE_TEST_SUITE_P(nightly_BehaviorTests_OVClassSeveralDevicesTest, OVClassSeveralDevicesTestCompileModel, + ::testing::Values(std::vector( + {std::string(ov::test::utils::DEVICE_NPU) + "." + + removeDeviceNameOnlyID(ov::test::utils::getTestsPlatformFromEnvironmentOr("3700"))})), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(nightly_BehaviorTests_OVClassModelOptionalTestP, OVClassModelOptionalTestP, + ::testing::Values(ov::test::utils::DEVICE_NPU), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P( + nightly_BehaviorTests_OVClassSeveralDevicesTest, OVClassSeveralDevicesTestQueryModel, + ::testing::Values(std::vector( + {std::string(ov::test::utils::DEVICE_NPU) + "." + + removeDeviceNameOnlyID(ov::test::utils::getTestsPlatformFromEnvironmentOr("3700")), + std::string(ov::test::utils::DEVICE_NPU) + "." + + removeDeviceNameOnlyID(ov::test::utils::getTestsPlatformFromEnvironmentOr("3700"))})), + (ov::test::utils::appendPlatformTypeTestName)); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_threading_tests.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_threading_tests.cpp new file mode 100644 index 00000000000..159ba271d91 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_threading_tests.cpp @@ -0,0 +1,46 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "behavior/ov_plugin/core_threading.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "npu_private_properties.hpp" + +namespace { + +const Params params[] = { + std::tuple{ov::test::utils::DEVICE_NPU, {{ov::enable_profiling(true)}}}, + std::tuple{ + ov::test::utils::DEVICE_HETERO, + {{ov::device::priorities(ov::test::utils::DEVICE_NPU, ov::test::utils::DEVICE_CPU)}}}, +}; + +const Params paramsStreams[] = { + std::tuple{ov::test::utils::DEVICE_NPU, {{ov::num_streams(ov::streams::AUTO)}}}, +}; + +const Params paramsStreamsDRIVER[] = { + std::tuple{ov::test::utils::DEVICE_NPU, + {{ov::num_streams(ov::streams::AUTO), + ov::intel_npu::compiler_type(ov::intel_npu::CompilerType::DRIVER)}}}, +}; + +} // namespace + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_CoreThreadingTest_NPU, CoreThreadingTest, testing::ValuesIn(params), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_CoreThreadingTest_NPU, CoreThreadingTestsWithIter, + testing::Combine(testing::ValuesIn(params), testing::Values(4), testing::Values(50)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_CoreThreadingTest_Streams_NPU, CoreThreadingTestsWithCacheEnabled, + testing::Combine(testing::ValuesIn(paramsStreamsDRIVER), testing::Values(20), + testing::Values(10)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_CoreThreadingTest_Streams_NPU, CoreThreadingTestsWithIter, + testing::Combine(testing::ValuesIn(paramsStreams), testing::Values(4), testing::Values(50)), + (ov::test::utils::appendPlatformTypeTestName)); diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/life_time.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/life_time.cpp new file mode 100644 index 00000000000..8a69e599872 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/life_time.cpp @@ -0,0 +1,48 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/life_time.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" +#include "overload/ov_plugin/life_time.hpp" + +using namespace ov::test::behavior; + +namespace { + +static std::string getTestCaseName(testing::TestParamInfo obj) { + std::string target_device = obj.param; + std::replace(target_device.begin(), target_device.end(), ':', '.'); + return "target_device=" + target_device + + "_targetPlatform=" + ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU) + "_"; +} + +const std::vector configs = {{}}; + +const std::vector device_names_and_priorities = { + "MULTI:NPU", // NPU via MULTI, + "AUTO:NPU", // NPU via AUTO, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVHoldersTest, ::testing::Values(ov::test::utils::DEVICE_NPU), + getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVHoldersTestOnImportedNetwork, + ::testing::Values(ov::test::utils::DEVICE_NPU), getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVHoldersTestNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + OVHoldersTestNPU::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVHoldersTestOnImportedNetworkNPU, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs)), + OVHoldersTestOnImportedNetworkNPU::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_VirtualPlugin_BehaviorTests, OVHoldersTestWithConfig, + ::testing::ValuesIn(device_names_and_priorities), + (ov::test::utils::appendPlatformTypeTestName)); +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp new file mode 100644 index 00000000000..bb7998d0be7 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp @@ -0,0 +1,247 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/properties_tests.hpp" +#include +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/al/config/common.hpp" +#include "npu_private_properties.hpp" +#include "openvino/runtime/intel_cpu/properties.hpp" +#include "openvino/runtime/intel_gpu/properties.hpp" +#include "openvino/runtime/intel_npu/properties.hpp" + +namespace { + +template +constexpr std::vector operator+(const std::vector& vector1, const std::vector& vector2) { + std::vector result; + result.insert(std::end(result), std::begin(vector1), std::end(vector1)); + result.insert(std::end(result), std::begin(vector2), std::end(vector2)); + return result; +} + +ov::log::Level getTestsLogLevelFromEnvironmentOr(ov::log::Level instead) { + if (auto var = std::getenv("OV_NPU_LOG_LEVEL")) { + std::istringstream stringStream = std::istringstream(var); + ov::log::Level level; + + stringStream >> level; + + return level; + } + return instead; +} + +const std::vector CorrectPluginMutableProperties = { + // OV + {{ov::hint::performance_mode.name(), ov::hint::PerformanceMode::THROUGHPUT}}, + {{ov::hint::num_requests.name(), 2u}}, + {{ov::log::level.name(), ov::log::Level::ERR}}, + {{ov::device::id.name(), removeDeviceNameOnlyID(ov::test::utils::getTestsPlatformFromEnvironmentOr("3700"))}}, + {{ov::enable_profiling.name(), true}}, +}; + +const std::vector CorrectPluginDefaultMutableProperties = { + /// OV + {{ov::enable_profiling.name(), false}}, + {{ov::hint::performance_mode.name(), ov::hint::PerformanceMode::LATENCY}}, + {{ov::hint::num_requests.name(), 1u}}, + {{ov::log::level.name(), getTestsLogLevelFromEnvironmentOr(ov::log::Level::NO)}}, + {{ov::device::id.name(), ""}}, + {{ov::num_streams.name(), ov::streams::Num(1)}}, +}; + +const std::vector ImmutableProperties{ov::supported_properties.name(), + ov::streams::num.name(), + ov::optimal_number_of_infer_requests.name(), + ov::intel_npu::device_alloc_mem_size.name(), + ov::intel_npu::device_total_mem_size.name(), + ov::intel_npu::driver_version.name(), + ov::available_devices.name(), + ov::device::capabilities.name(), + ov::range_for_async_infer_requests.name(), + ov::range_for_streams.name(), + ov::device::uuid.name(), + ov::device::architecture.name(), + ov::device::full_name.name() + +}; + +const std::vector CorrectCompiledModelProperties = { + {{ov::device::id.name(), removeDeviceNameOnlyID(ov::test::utils::getTestsPlatformFromEnvironmentOr("3700"))}}, + {{ov::enable_profiling.name(), true}}, + {{ov::hint::performance_mode.name(), ov::hint::PerformanceMode::LATENCY}}, + {{ov::hint::num_requests.name(), 4u}}, + {{ov::hint::enable_cpu_pinning.name(), true}}, +}; + +const std::vector IncorrectImmutableProperties = { + {{ov::streams::num.name(), ov::streams::Num(2)}}, + {{ov::optimal_number_of_infer_requests.name(), 4}}, + {{ov::intel_npu::device_alloc_mem_size.name(), 1024}}, + {{ov::intel_npu::device_total_mem_size.name(), 2048}}, + {{ov::intel_npu::driver_version.name(), 3}}, + {{ov::available_devices.name(), testing::internal::Strings{"3700", "3720"}}}, + {{ov::device::capabilities.name(), testing::internal::Strings{ov::device::capability::BF16}}}, + {{ov::range_for_async_infer_requests.name(), + std::tuple{1u, 10u, 1u}}}, + {{ov::range_for_streams.name(), std::tuple{1u, 4u}}}, + {{ov::device::uuid.name(), + ov::device::UUID{std::array{ + 0xAB, 0xCD, 0xEF, 0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x12, 0x34, 0x56, 0x67}}}}, + {{ov::device::architecture.name(), "3720"}}, + {{ov::device::full_name.name(), "NPU.3700"}}}; // namespace + +const std::vector IncorrectMutablePropertiesWrongValueTypes = { + // OV + {{ov::enable_profiling.name(), 'c'}}, + {{ov::hint::performance_mode.name(), ov::intel_npu::CompilerType::DRIVER}}, + {{ov::hint::num_requests.name(), -2.0f}}, + {{ov::log::level.name(), -2}}, + {{ov::device::id.name(), "false"}}, + {{ov::num_streams.name(), "one"}}, +}; + +const std::vector IncorrectInexistingProperties = { + {{ov::affinity.name(), ov::Affinity::HYBRID_AWARE}}, + {{ov::intel_cpu::denormals_optimization.name(), true}}, + {{ov::intel_gpu::hint::host_task_priority.name(), ov::hint::Priority::LOW}}, + {{ov::intel_gpu::hint::queue_throttle.name(), ov::intel_gpu::hint::ThrottleLevel::HIGH}}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVPropertiesTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(CorrectPluginMutableProperties)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(IncorrectImmutableProperties + + IncorrectMutablePropertiesWrongValueTypes + + IncorrectInexistingProperties)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVPropertiesDefaultSupportedTests, + ::testing::Values(ov::test::utils::DEVICE_NPU), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassCommon, OVBasicPropertiesTestsP, + ::testing::Values(std::make_pair("openvino_intel_npu_plugin", ov::test::utils::DEVICE_NPU)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVPropertiesDefaultTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(CorrectPluginDefaultMutableProperties)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests_OVCheckSetSupportedRWMetricsPropsTests, OVCheckSetSupportedRWMetricsPropsTests, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(getRWMandatoryPropertiesValues(CorrectPluginMutableProperties))), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests_OVCheckGetSupportedROMetricsPropsTests, OVCheckGetSupportedROMetricsPropsTests, + ::testing::Combine( + ::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(OVCheckGetSupportedROMetricsPropsTests::configureProperties(ImmutableProperties))), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests_OVCheckChangePropComplieModleGetPropTests_DEVICE_ID, + OVCheckChangePropComplieModleGetPropTests_DEVICE_ID, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(CorrectCompiledModelProperties)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests_OVCheckChangePropComplieModleGetPropTests_InferencePrecision, + OVCheckChangePropComplieModleGetPropTests_InferencePrecision, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(CorrectCompiledModelProperties)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVCheckMetricsPropsTests_ModelDependceProps, + OVCheckMetricsPropsTests_ModelDependceProps, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(CorrectCompiledModelProperties)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassSetDefaultDeviceIDPropTest, OVClassSetDefaultDeviceIDPropTest, + ::testing::Values(std::make_pair( + ov::test::utils::DEVICE_NPU, + removeDeviceNameOnlyID(ov::test::utils::getTestsPlatformFromEnvironmentOr("3700")))), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests_OVClassSpecificDeviceTest, OVSpecificDeviceSetConfigTest, + ::testing::Values(std::string(ov::test::utils::DEVICE_NPU) + "." + + removeDeviceNameOnlyID(ov::test::utils::getTestsPlatformFromEnvironmentOr("3700"))), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests_OVClassSpecificDeviceTest, OVSpecificDeviceGetConfigTest, + ::testing::Values(std::string(ov::test::utils::DEVICE_NPU) + "." + + removeDeviceNameOnlyID(ov::test::utils::getTestsPlatformFromEnvironmentOr("3700"))), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVGetAvailableDevicesPropsTest, OVGetAvailableDevicesPropsTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests_OVClassSpecificDeviceTest, OVSpecificDeviceTestSetConfig, + ::testing::Values(std::string(ov::test::utils::DEVICE_NPU) + "." + + removeDeviceNameOnlyID(ov::test::utils::getTestsPlatformFromEnvironmentOr("3700"))), + (ov::test::utils::appendPlatformTypeTestName)); + +const std::vector multiConfigs = {{ov::device::priorities(ov::test::utils::DEVICE_NPU)}}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassSetDevicePriorityConfigPropsTest, + OVClassSetDevicePriorityConfigPropsTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_HETERO), + ::testing::ValuesIn(multiConfigs)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVGetMetricPropsTest, OVGetMetricPropsTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVGetMetricPropsOptionalTest, OVGetMetricPropsOptionalTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + (ov::test::utils::appendPlatformTypeTestName)); + +const std::vector configsDeviceProperties = { + {ov::device::properties(ov::test::utils::DEVICE_NPU, ov::num_streams(ov::streams::AUTO))}, + {ov::device::properties( + ov::AnyMap{{ov::test::utils::DEVICE_NPU, ov::AnyMap{ov::num_streams(ov::streams::AUTO)}}})}}; + +const std::vector configsDevicePropertiesDouble = { + {ov::device::properties(ov::test::utils::DEVICE_NPU, ov::num_streams(-1)), ov::num_streams(ov::streams::AUTO)}, + {ov::device::properties(ov::test::utils::DEVICE_NPU, ov::num_streams(-1)), + ov::device::properties( + ov::AnyMap{{ov::test::utils::DEVICE_NPU, ov::AnyMap{ov::num_streams(ov::streams::AUTO)}}}), + ov::num_streams(-1)}, + {ov::device::properties(ov::test::utils::DEVICE_NPU, ov::num_streams(-1)), + ov::device::properties(ov::test::utils::DEVICE_NPU, ov::num_streams(ov::streams::AUTO))}, + {ov::device::properties(ov::test::utils::DEVICE_NPU, ov::num_streams(ov::streams::AUTO)), + ov::device::properties(ov::AnyMap{{ov::test::utils::DEVICE_NPU, ov::AnyMap{ov::num_streams(-1)}}})}, + {ov::device::properties( + ov::AnyMap{{ov::test::utils::DEVICE_NPU, ov::AnyMap{ov::num_streams(ov::streams::AUTO)}}})}}; + +// IE Class load and check network with ov::device::properties +INSTANTIATE_TEST_SUITE_P( + smoke_NPU_BehaviorTests_OVClassCompileModelAndCheckSecondaryPropertiesTest, + OVClassCompileModelAndCheckSecondaryPropertiesTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configsDeviceProperties)), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P( + smoke_NPU_BehaviorTests_OVClassCompileModelAndCheckWithSecondaryPropertiesDoubleTest, + OVClassCompileModelAndCheckSecondaryPropertiesTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configsDevicePropertiesDouble)), + (ov::test::utils::appendPlatformTypeTestName)); +}; // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/query_model.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/query_model.cpp new file mode 100644 index 00000000000..339fa35cd9b --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/query_model.cpp @@ -0,0 +1,23 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/query_model.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" + +namespace ov { +namespace test { +namespace behavior { + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassModelTestP, OVClassModelTestP, + ::testing::Values(ov::test::utils::DEVICE_NPU), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassQueryModelTestTests, OVClassQueryModelTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + (ov::test::utils::appendPlatformTypeTestName)); + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/remote.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/remote.cpp new file mode 100644 index 00000000000..3ef0cb3ba0d --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/remote.cpp @@ -0,0 +1,27 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/remote.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "common_test_utils/test_constants.hpp" + +using namespace ov::test; + +namespace { +const std::vector configs = {{}}; + +std::vector> generate_remote_params() { + return {{}}; +} + +// [Tracking number: E#110088] +// NPU plugin does not support `Remote Tensors` yet +INSTANTIATE_TEST_SUITE_P(DISABLED_TMP_smoke_BehaviorTests, OVRemoteTest, + ::testing::Combine(::testing::Values(ov::element::f32), + ::testing::Values(::ov::test::utils::DEVICE_NPU), + ::testing::ValuesIn(configs), + ::testing::ValuesIn(generate_remote_params())), + (ov::test::utils::appendPlatformTypeTestName)); +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/version.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/version.cpp new file mode 100644 index 00000000000..020901dce6a --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/version.cpp @@ -0,0 +1,21 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/version.hpp" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" + +namespace ov { +namespace test { +namespace behavior { + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, VersionTests, ::testing::Values(ov::test::utils::DEVICE_NPU), + (ov::test::utils::appendPlatformTypeTestName)); + +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, VersionTests, ::testing::Values(ov::test::utils::DEVICE_HETERO), + (ov::test::utils::appendPlatformTypeTestName)); + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_report.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_report.cpp new file mode 100644 index 00000000000..353a3af0f9c --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_report.cpp @@ -0,0 +1,75 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "npu_test_report.hpp" +#include "functional_test_utils/summary/op_summary.hpp" + +#include +#include + +namespace ov::test::utils { + +const std::array(VpuTestStage::LAST_VALUE)> VpuTestReport::stages = { + "RUN", "COMPILED", "IMPORTED", "INFERRED", "VALIDATED", "SKIPPED_EXCEPTION"}; + +std::string testName(const testing::TestInfo* testInfo); + +std::string testName(const testing::TestInfo* testInfo) { + const std::string name(testInfo->test_case_name()); + auto npos = name.find("/"); + + return (npos != std::string::npos) ? name.substr(npos + 1) : name; +} + +VpuTestReport::VpuTestReport() { +} + +void VpuTestReport::run(const testing::TestInfo* testInfo) { + ++counters[testName(testInfo)][static_cast(VpuTestStage::RUN)]; +} + +void VpuTestReport::compiled(const testing::TestInfo* testInfo) { + ++counters[testName(testInfo)][static_cast(VpuTestStage::COMPILED)]; +} + +void VpuTestReport::imported(const testing::TestInfo* testInfo) { + ++counters[testName(testInfo)][static_cast(VpuTestStage::IMPORTED)]; +} + +void VpuTestReport::inferred(const testing::TestInfo* testInfo) { + ++counters[testName(testInfo)][static_cast(VpuTestStage::INFERRED)]; +} + +void VpuTestReport::validated(const testing::TestInfo* testInfo) { + ++counters[testName(testInfo)][static_cast(VpuTestStage::VALIDATED)]; +} + +void VpuTestReport::skipped(const testing::TestInfo* testInfo) { + ++counters[testName(testInfo)][static_cast(VpuTestStage::SKIPPED_EXCEPTION)]; +} + +void NpuTestReportEnvironment::TearDown() { + std::cout << "TestReportResult: " << std::endl; + const auto& counters = VpuTestReport::getInstance().getCounters(); + + std::array(VpuTestStage::LAST_VALUE)> totals = {}; + for (auto const& cit : counters) { + std::cout << cit.first << ": "; + for (int it = static_cast(VpuTestStage::RUN); it < static_cast(VpuTestStage::LAST_VALUE); ++it) { + totals[it] += cit.second[it]; + std::cout << VpuTestReport::stages[it] << " - " << cit.second[it] << "; "; + } + std::cout << std::endl; + } + + std::cout << "VpuTotalTestCases: "; + for (int it = static_cast(VpuTestStage::RUN); it < static_cast(VpuTestStage::LAST_VALUE); ++it) { + std::cout << VpuTestReport::stages[it] << " - " << totals[it] << "; "; + } + std::cout << std::endl; + + ov::test::utils::OpSummary::getInstance().saveReport(); +}; + +} // namespace ov::test::utils diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_report.hpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_report.hpp new file mode 100644 index 00000000000..d65a7c32c8b --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_report.hpp @@ -0,0 +1,50 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +namespace ov::test::utils { + +enum class VpuTestStage { RUN = 0, COMPILED, IMPORTED, INFERRED, VALIDATED, SKIPPED_EXCEPTION, LAST_VALUE }; + +class VpuTestReport { +private: + std::map(VpuTestStage::LAST_VALUE)>> counters; + +public: + static const std::array(VpuTestStage::LAST_VALUE)> stages; + +public: + explicit VpuTestReport(); + void run(const testing::TestInfo* testInfo); + void compiled(const testing::TestInfo* testInfo); + void imported(const testing::TestInfo* testInfo); + void inferred(const testing::TestInfo* testInfo); + void validated(const testing::TestInfo* testInfo); + void skipped(const testing::TestInfo* testInfo); + + const std::map(VpuTestStage::LAST_VALUE)>>& getCounters() const { + return counters; + } + + static VpuTestReport& getInstance() { + static VpuTestReport instance; + return instance; + } +}; + +class NpuTestReportEnvironment : public testing::Environment { +public: + ~NpuTestReportEnvironment() override { + } + + void TearDown() override; +}; + +} // namespace ov::test::utils diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_tool.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_tool.cpp new file mode 100644 index 00000000000..e5332665df7 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_tool.cpp @@ -0,0 +1,26 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "npu_test_tool.hpp" +#include + +#include +#include +#include + +namespace ov::test::utils { + +NpuTestTool::NpuTestTool(const NpuTestEnvConfig& envCfg) + : envConfig(envCfg), + DEVICE_NAME(envConfig.IE_NPU_TESTS_DEVICE_NAME.empty() ? "NPU" : envConfig.IE_NPU_TESTS_DEVICE_NAME), + _log("NpuTestTool", ov::log::Level::INFO) { +} + +std::string NpuTestTool::getDeviceMetric(std::string name) { + std::shared_ptr core = ov::test::utils::PluginCache::get().core(DEVICE_NAME); + + return core->get_property(DEVICE_NAME, name).as(); +} + +} // namespace ov::test::utils diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_tool.hpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_tool.hpp new file mode 100644 index 00000000000..a4a3a34661f --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/npu_test_tool.hpp @@ -0,0 +1,29 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include "common/npu_test_env_cfg.hpp" +#include "intel_npu/utils/logger/logger.hpp" + +namespace ov::test::utils { + +class NpuTestTool { +public: + const NpuTestEnvConfig& envConfig; + const std::string DEVICE_NAME; + ::intel_npu::Logger _log; + +public: + explicit NpuTestTool(const NpuTestEnvConfig& envCfg); + + std::string getDeviceMetric(std::string name); +}; + +} // namespace ov::test::utils diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/skip_tests_config.cpp new file mode 100644 index 00000000000..18c6582ca35 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -0,0 +1,972 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "functional_test_utils/skip_tests_config.hpp" + +#include +#include +#include + +#include "common/functions.h" +#include "common/utils.hpp" +#include "common/npu_test_env_cfg.hpp" +#include "common_test_utils/common_utils.hpp" +#include "functional_test_utils/ov_plugin_cache.hpp" +#include +#include "npu_private_properties.hpp" + +class BackendName { +public: + BackendName() { + const auto corePtr = ov::test::utils::PluginCache::get().core(); + if (corePtr != nullptr) { + _name = getBackendName(*corePtr); + } else { + _log.error("Failed to get OpenVINO Core from cache!"); + } + } + + std::string getName() const { + return _name; + } + + bool isEmpty() const noexcept { + return _name.empty(); + } + + bool isZero() const { + return _name == "LEVEL0"; + } + + bool isVpual() const { + return _name == "VPUAL"; + } + + bool isIMD() const { + return _name == "IMD"; + } + +private: + std::string _name; + intel_npu::Logger _log = intel_npu::Logger("BackendName", ov::log::Level::INFO); +}; + +class AvailableDevices { +public: + AvailableDevices() { + const auto corePtr = ov::test::utils::PluginCache::get().core(); + if (corePtr != nullptr) { + _availableDevices = ::getAvailableDevices(*corePtr); + } else { + _log.error("Failed to get OpenVINO Core from cache!"); + } + + // Private device names may be registered via environment variables + const std::string environmentDevice = ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::intel_npu::Platform::AUTO_DETECT.data()); + const std::string standardizedEnvironmentDevice = ov::intel_npu::Platform::standardize(environmentDevice); + + if (std::all_of(_availableDevices.begin(), _availableDevices.end(), [&](const std::string& deviceName) { + return deviceName.find(standardizedEnvironmentDevice) == std::string::npos; + })) { + _availableDevices.push_back(standardizedEnvironmentDevice); + } + } + + const auto& getAvailableDevices() const { + return _availableDevices; + } + + auto count() const { + return _availableDevices.size(); + } + + bool has3720() const { + return std::any_of(_availableDevices.begin(), _availableDevices.end(), [](const std::string& deviceName) { + return deviceName.find("3720") != std::string::npos; + }); + } + + bool has3700() const { + return std::any_of(_availableDevices.begin(), _availableDevices.end(), [](const std::string& deviceName) { + return deviceName.find("3700") != std::string::npos; + }); + } + +private: + std::vector _availableDevices; + intel_npu::Logger _log = intel_npu::Logger("AvailableDevices", ov::log::Level::INFO); +}; + +class SkipRegistry { +public: + void addPatterns(std::string&& comment, std::vector&& patternsToSkip) { + _registry.emplace_back(std::move(comment), std::move(patternsToSkip)); + } + + void addPatterns(bool conditionFlag, std::string&& comment, std::vector&& patternsToSkip) { + if (conditionFlag) { + addPatterns(std::move(comment), std::move(patternsToSkip)); + } + } + + /** Searches for the skip pattern to which passed test name matches. + * Prints the message onto console if pattern is found and the test is to be skipped + * + * @param testName name of the current test being matched against skipping + * @return Suitable skip pattern or empty string if none + */ + std::string getMatchingPattern(const std::string& testName) const { + for (const auto& entry : _registry) { + for (const auto& pattern : entry._patterns) { + std::regex re(pattern); + if (std::regex_match(testName, re)) { + _log.info("%s; Pattern: %s", entry._comment.c_str(), pattern.c_str()); + return pattern; + } + } + } + + return std::string{}; + } + +private: + struct Entry { + Entry(std::string&& comment, std::vector&& patterns) + : _comment{std::move(comment)}, _patterns{std::move(patterns)} { + } + + std::string _comment; + std::vector _patterns; + }; + + std::vector _registry; + intel_npu::Logger _log = intel_npu::Logger("SkipRegistry", ov::log::Level::INFO); +}; + +std::string getCurrentTestName(); + +std::string getCurrentTestName() { + const auto* currentTestInfo = ::testing::UnitTest::GetInstance()->current_test_info(); + const auto currentTestName = currentTestInfo->test_case_name() + std::string(".") + currentTestInfo->name(); + return currentTestName; +} + +std::vector disabledTestPatterns(); + +std::vector disabledTestPatterns() { + // Initialize skip registry + static const auto skipRegistry = []() { + SkipRegistry _skipRegistry; + + const BackendName backendName; + const AvailableDevices devices; + + // clang-format off + + // + // Disabled test patterns + // + // TODO + _skipRegistry.addPatterns( + "Tests break due to starting infer on IA side", { + ".*CorrectConfigAPITests.*", + }); + + _skipRegistry.addPatterns( + "ARM CPU Plugin is not available on Yocto", { + ".*IEClassLoadNetworkTest.*HETERO.*", + ".*IEClassLoadNetworkTest.*MULTI.*", + }); + + // TODO + // [Track number: E#30810] + _skipRegistry.addPatterns( + "Hetero plugin doesn't throw an exception in case of big device ID", { + ".*OVClassLoadNetworkTestNPU.*LoadNetworkHETEROWithBigDeviceIDThrows.*", + }); + + // TODO + // [Track number: E#30815] + _skipRegistry.addPatterns( + "NPU Plugin doesn't handle DEVICE_ID in QueryNetwork implementation", { + ".*OVClassQueryNetworkTest.*", + }); + + // [Track number: E#12774] + _skipRegistry.addPatterns( + "Cannot detect npu platform when it's not passed; Skip tests on Yocto which passes device without platform", { + ".*IEClassLoadNetworkTest.LoadNetworkWithDeviceIDNoThrow.*", + ".*IEClassLoadNetworkTest.LoadNetworkWithBigDeviceIDThrows.*", + ".*IEClassLoadNetworkTest.LoadNetworkWithInvalidDeviceIDThrows.*", + }); + + // [Track number: E#28335] + _skipRegistry.addPatterns( + "Disabled test E#28335", { + ".*smoke_LoadNetworkToDefaultDeviceNoThrow.*", + }); + + // [Track number: E#32241] + _skipRegistry.addPatterns( + "Disabled test E#28335", { + ".*LoadNetwork.*CheckDeviceInBlob.*", + }); + + // [Track number: S#27343] + _skipRegistry.addPatterns( + "double free detected", { + ".*InferConfigInTests\\.CanInferWithConfig.*", + }); + + // TODO: + _skipRegistry.addPatterns( + "GetExecGraphInfo function is not implemented for NPU plugin", { + ".*checkGetExecGraphInfoIsNotNullptr.*", + ".*CanCreateTwoExeNetworksAndCheckFunction.*", + ".*CanCreateTwoCompiledModelsAndCheckRuntimeModel.*", + ".*CheckExecGraphInfo.*", + ".*canLoadCorrectNetworkToGetExecutable.*", + }); + + // [Track number: E#31074] + _skipRegistry.addPatterns( + "Disabled test E#28335", { + ".*checkInferTime.*", + ".*OVExecGraphImportExportTest.*", + }); + + _skipRegistry.addPatterns( + "Test uses legacy OpenVINO 1.0 API, no need to support it", { + ".*ExecutableNetworkBaseTest.checkGetMetric.*", + }); + + // TODO: + _skipRegistry.addPatterns( + "SetConfig function is not implemented for ExecutableNetwork interface (implemented only for npu plugin)", { + ".*ExecutableNetworkBaseTest.canSetConfigToExecNet.*", + ".*ExecutableNetworkBaseTest.canSetConfigToExecNetAndCheckConfigAndCheck.*", + ".*CanSetConfigToExecNet.*", + }); + + // TODO + // [Track number: E#30822] + _skipRegistry.addPatterns( + "Exception 'Not implemented'", { + ".*OVClassNetworkTestP.*LoadNetworkCreateDefaultExecGraphResult.*", + }); + + _skipRegistry.addPatterns( + "This is openvino specific test", { + ".*ExecutableNetworkBaseTest.canExport.*", + }); + + _skipRegistry.addPatterns( + "TensorIterator layer is not supported", { + ".*ReturnResultNotReadyFromWaitInAsyncModeForTooSmallTimeout.*", + ".*OVInferRequestDynamicTests.*", + ".*OVInferenceChaining.*", + }); + + _skipRegistry.addPatterns( + "Tests with unsupported precision", { + ".*InferRequestCheckTensorPrecision.*type=boolean.*", + ".*InferRequestCheckTensorPrecision.*type=bf16.*", + ".*InferRequestCheckTensorPrecision.*type=f64.*", + ".*InferRequestCheckTensorPrecision.*type=i4.*", + ".*InferRequestCheckTensorPrecision.*type=u4.*", + ".*InferRequestCheckTensorPrecision.*type=u1\\D.*", + // [Track number: E#97469] + ".*InferRequestCheckTensorPrecision.*type=i64.*", + }); + + _skipRegistry.addPatterns(!backendName.isZero() || !devices.has3720(), + "Tests enabled only for L0 NPU3720", { + // [Track number: E#70764] + ".*InferRequestCheckTensorPrecision.*", + ".*InferRequestIOTensorSetPrecisionTest.*", + ".*DriverCompilerAdapterDowngradeInterpolate11TestNPU.*", + ".*DriverCompilerAdapterInputsOutputsTestNPU.*", + }); + + // TODO + // [Track number: E#32075] + _skipRegistry.addPatterns( + "Exception during loading to the device", { + ".*OVClassLoadNetworkTestNPU.*LoadNetworkHETEROwithMULTINoThrow.*", + ".*OVClassLoadNetworkTestNPU.*LoadNetworkMULTIwithHETERONoThrow.*", + }); + + _skipRegistry.addPatterns( + "compiler: Unsupported arch kind: NPUX311X", { + ".*CompilationForSpecificPlatform.*(3800|3900).*", + }); + + // [Track number: E#67741] + _skipRegistry.addPatterns( + "Cannot call setShape for Blobs", { + R"(.*(smoke_Behavior|smoke_Auto_Behavior|smoke_Multi_Behavior).*OVInferRequestIOTensorTest.*canInferAfterIOBlobReallocation.*)", + R"(.*(smoke_Behavior|smoke_Auto_Behavior|smoke_Multi_Behavior).*OVInferRequestIOTensorTest.*InferStaticNetworkSetChangedInputTensorThrow.*targetDevice=(NPU_|MULTI_configItem=MULTI_DEVICE_PRIORITIES_NPU).*)" + }); + + // [Track number: E#67749] + _skipRegistry.addPatterns( + "Can't loadNetwork without cache for ReadConcatSplitAssign with precision f32", { + ".*CachingSupportCase_NPU.*CompileModelCacheTestBase.*CompareWithRefImpl.*ReadConcatSplitAssign.*", + }); + + // [Tracking number: E#99817] + _skipRegistry.addPatterns( + "NPU Plugin currently fails to get a valid output in these test cases", { + ".*OVInferRequestIOTensorTest.InferStaticNetworkSetChangedInputTensorThrow.*", + ".*OVInferRequestIOTensorTestNPU.InferStaticNetworkSetChangedInputTensorThrow.*", + R"(.*OVInferRequestIOTensorTestNPU.InferStaticNetworkSetChangedInputTensorThrow/targetDevice=NPU3720_.*)", + R"(.*OVInferRequestIOTensorTestNPU.InferStaticNetworkSetChangedInputTensorThrow/targetDevice=NPU3720_configItem=MULTI_DEVICE_PRIORITIES_NPU_.*)", + R"(.*OVInferRequestIOTensorTest.InferStaticNetworkSetChangedInputTensorThrow/targetDevice=NPU3720_.*)", + R"(.*OVInferRequestIOTensorTest.InferStaticNetworkSetChangedInputTensorThrow/targetDevice=NPU3720_configItem=MULTI_DEVICE_PRIORITIES_NPU_.*)", + }); + + // [Track number: E#68774] + _skipRegistry.addPatterns( + "OV requires the plugin to throw when value of DEVICE_ID is unrecognized, but plugin does not throw", { + "smoke_BehaviorTests.*IncorrectConfigTests.SetConfigWithIncorrectKey.*(SOME_DEVICE_ID|DEVICE_UNKNOWN).*", + "smoke_BehaviorTests.*IncorrectConfigTests.SetConfigWithNoExistingKey.*SOME_DEVICE_ID.*", + "smoke_BehaviorTests.*IncorrectConfigAPITests.SetConfigWithNoExistingKey.*(SOME_DEVICE_ID|DEVICE_UNKNOWN).*", + }); + + // [Track number: E#77755] + _skipRegistry.addPatterns( + "OV requires the plugin to throw on network load when config file is incorrect, but plugin does not throw", { + R"(.*smoke_Auto_BehaviorTests.*IncorrectConfigTests.CanNotLoadNetworkWithIncorrectConfig.*AUTO_config.*unknown_file_MULTI_DEVICE_PRIORITIES=(NPU_|NPU,CPU_).*)" + }); + + // [Track number: E#77756] + _skipRegistry.addPatterns( + "OV expects the plugin to not throw any exception on network load, but it actually throws", { + R"(.*(smoke_Multi_Behavior|smoke_Auto_Behavior).*SetPropLoadNetWorkGetPropTests.*SetPropLoadNetWorkGetProperty.*)" + }); + + // [Track number: E#68776] + _skipRegistry.addPatterns( + "Plugin can not perform SetConfig for value like: device=NPU config key=LOG_LEVEL value=0", { + "smoke_BehaviorTests/DefaultValuesConfigTests.CanSetDefaultValueBackToPlugin.*", + }); + + // [Track number: E#80555] + _skipRegistry.addPatterns(devices.has3700(), + "Problems with SplitConcat ngraph function", { + R"(.*smoke_BehaviorTests/InferRequest(CallbackTests|MultithreadingTests|PerfCountersTest|WaitTests)\..*)", + R"(.*smoke_BehaviorTests(/|/OV)InferRequestCancellationTests\..*)", + R"(.*smoke(_|_Multi_)BehaviorTests/OVInferRequestIOTensorTest\..*)", + R"(.*smoke(_|_Auto_|_Multi_)BehaviorTests/OVInferRequest(CallbackTests|IOTensorSetPrecisionTest|MultithreadingTests)\..*)", + ".*OVClassNetworkTestP.LoadNetworkActual.*", + ".*OVClassLoadNetworkTestNPU.LoadNetworkHETEROWithDeviceIDNoThrow.*", + R"(.*OVHoldersTest\..*)", + R"(.*OVHoldersTestOnImportedNetwork\..*)", + }); + + _skipRegistry.addPatterns( + "Disabled with ticket number", { + // [Track number: E#48480] + ".*OVExecutableNetworkBaseTest.*", + + // [Track number: E#63708] + ".*smoke_BehaviorTests.*InferStaticNetworkSetInputTensor.*", + ".*smoke_Multi_BehaviorTests.*InferStaticNetworkSetInputTensor.*", + + // [Track number: E#64490] + ".*OVClassNetworkTestP.*SetAffinityWithConstantBranches.*" + }); + + // [Tracking number: E#86380] + _skipRegistry.addPatterns( + "The output tensor gets freed when the inference request structure's destructor is called. The issue is unrelated to the caching feature.", { + ".*CacheTestBase.CompareWithRefImpl.*", + }); + + _skipRegistry.addPatterns( + "Expected: ie->SetConfig(configuration, target_device) throws an exception of type InferenceEngine::Exception. Throws nothing.", { + // [Tracking number: E#89274] + ".*AutoBatch.*Behavior.*IncorrectConfigAPITests.SetConfigWithNoExistingKey.*AUTO_BATCH_TIMEOUT.*", + // [Track number: E#89084] + ".*AutoBatch.*Behavior.*IncorrectConfigTests.SetConfigWithIncorrectKey.*AUTO_BATCH_TIMEOUT.*", + ".*AutoBatch.*Behavior.*IncorrectConfigTests.CanNotLoadNetworkWithIncorrectConfig.*AUTO_BATCH_TIMEOUT.*", + }); + + _skipRegistry.addPatterns( + "Dynamic I/O shapes are being used when running the tests. This feature is not yet supported by the NPU plugin.", { + ".*SetPreProcessTo.*" + }); + + _skipRegistry.addPatterns( + "This scenario became invalid upon refactoring the implementation as to use the 2.0 OV API. " + "The legacy version structure contains major and minor version attributes, but these fields are not found anymore " + "in the corresponding 2.0 API structure.", { + ".*smoke_BehaviorTests/VersionTest.pluginCurrentVersionIsCorrect.*" + }); + + // [Tracking number: E#102428] + _skipRegistry.addPatterns( + "Tests throw errors as expected but drivers post-v.1657 will fail to catch them", { + ".*FailGracefullyTest.*", + ".*QueryNetworkTestSuite3NPU.*" + }); + + // + // Conditionally disabled test patterns + // + + _skipRegistry.addPatterns(devices.count() && !devices.has3720(), "Tests are disabled for all devices except NPU3720", + { + // [Track number: E#49620] + ".*NPU3700(\\.|_)(SW|HW).*", + ".*NPU3720.*", + // [Track number: E#84621] + ".*DriverCompilerAdapterDowngradeInterpolate11TestNPU.*", + ".*DriverCompilerAdapterInputsOutputsTestNPU.*", + }); + + _skipRegistry.addPatterns( + backendName.isEmpty(), "Disabled for when backend is empty (i.e., no device)", + { + // Cannot run InferRequest tests without a device to infer to + ".*InferRequest.*", + ".*OVInferRequest.*", + ".*OVInferenceChaining.*", + ".*ExecutableNetworkBaseTest.*", + ".*OVExecutableNetworkBaseTest.*", + ".*ExecNetSetPrecision.*", + ".*SetBlobTest.*", + ".*InferRequestCallbackTests.*", + ".*PreprocessingPrecisionConvertTest.*", + ".*SetPreProcessToInputInfo.*", + ".*InferRequestPreprocess.*", + ".*HoldersTestOnImportedNetwork.*", + ".*HoldersTest.Orders.*", + ".*HoldersTestImportNetwork.Orders.*", + + // Cannot compile network without explicit specifying of the platform in case of no devices + ".*OVExecGraphImportExportTest.*", + ".*OVHoldersTest.*", + ".*OVClassExecutableNetworkGetMetricTest.*", + ".*OVClassExecutableNetworkGetConfigTest.*", + ".*OVClassNetworkTestP.*SetAffinityWithConstantBranches.*", + ".*OVClassNetworkTestP.*SetAffinityWithKSO.*", + ".*OVClassNetworkTestP.*LoadNetwork.*", + ".*FailGracefullyTest.*", + ".*DriverCompilerAdapterInputsOutputsTestNPU.*", + + // Exception in case of network compilation without devices in system + // [Track number: E#30824] + ".*OVClassImportExportTestP.*", + ".*OVClassLoadNetworkTestNPU.*LoadNetwork.*", + // [Track number: E#84621] + ".*DriverCompilerAdapterDowngradeInterpolate11TestNPU.*", + ".*QueryNetworkTestSuite.*", + }); + + // [Tracking number: E#111510] + _skipRegistry.addPatterns("Failing test for NPU device", { + ".*OVClassImportExportTestP.*OVClassCompiledModelImportExportTestP.*ImportNetworkThrowWithDeviceName.*" + }); + + // [Track number: S#14836] + _skipRegistry.addPatterns(devices.has3700(), + "Async tests break on dKMB", { + ".*ExclusiveAsyncRequests.*", + }); + + _skipRegistry.addPatterns(backendName.isZero() && devices.has3700(), + "TensorIterator layer is not supported by dKMB platform", + { + ".*SetBlobTest.*", + }); + + _skipRegistry.addPatterns(backendName.isZero() && devices.has3700(), "Convert layer is not supported by dKMB platform", + {".*PreprocessingPrecisionConvertTest.*", ".*InferRequestPreprocess.*"}); + + _skipRegistry.addPatterns(backendName.isZero() && devices.has3700(), + "Tests fail on RPL dKMB boards, start_async() fails intermittently", + { // [Tracking number: E#90056] + ".*OVInferConsistencyTest.*", + // [Tracking number: E#92317] + ".*OVInferRequestIOTensorTest.*", + ".*OVInferRequestMultithreadingTests.*", + ".*OVInferRequestCallbackTests.*", + ".*InferRequestMultithreadingTests.*", + ".*InferRequestPerfCountersTest.*", + ".*InferRequestWaitTests.*", + ".*OVInferRequestCancellationTest.*", + ".*InferRequestRunTests.*", + ".*InferRequestCallbackTests.*", + ".*InferRequestCancellationTests.*", + ".*OVCompileAndInferRequest.*"}); + + _skipRegistry.addPatterns(!(backendName.isZero()), "These tests runs only on LevelZero backend", + {".*InferRequestRunTests.*", + ".*OVClassGetMetricAndPrintNoThrow.*", + ".*IEClassGetMetricAndPrintNoThrow.*", + ".*CompileModelLoadFromFileTestBase.*", + ".*CorrectConfigTests.*"}); + + _skipRegistry.addPatterns(!(devices.has3720()), "Runs only on NPU3720 with Level Zero enabled #85493", + {".*InferRequestRunTests.MultipleExecutorStreamsTestsSyncInfers.*"}); + + _skipRegistry.addPatterns("Other devices than NPU doesn't allow to set NPU properties with OV1.0 and CACHE_DIR + MLIR is not supported", + {".*smoke_AutoBatch_BehaviorTests/CorrectConfigTests.*"}); + + _skipRegistry.addPatterns("OpenVINO issues when using caching mechanism", + {// [Tracking number: CVS#119359] + ".*smoke_Auto_BehaviorTests_CachingSupportCase_NPU/CompileModelLoadFromFileTestBase.*", + // [Tracking number: CVS#120240] + ".*smoke_BehaviorTests_CachingSupportCase_NPU/CompileModelLoadFromFileTestBase.*"}); + + _skipRegistry.addPatterns(devices.has3700(), "Do not run the tests that require a new CiD version on the 3700 platform", + { + // [Tracking number: E#92279] + ".*CompileModelLoadFromFileTestBase.*", + ".*CorrectConfigTests.CanUseCache.*", + ".*CorrectConfigTests.CanLoadNetworkWithCorrectConfig.*", + ".*DriverCompilerAdapterDowngradeInterpolate11TestNPU.CheckOpsetVersion.*", + ".*DriverCompilerAdapterInputsOutputsTestNPU.CheckInOutputs.*", + ".*DriverCompilerAdapterExpectedThrowNPU.CheckWrongGraphExtAndThrow.*"}); + +#ifdef WIN32 +#elif defined(__linux__) + // [Tracking number: E#103391] + _skipRegistry.addPatterns(backendName.isZero() && devices.has3720(), + "IfTest segfaults npuFuncTest on Ubuntu", { + ".*smoke_IfTest.*" + }); + + _skipRegistry.addPatterns(backendName.isZero() && devices.has3720(), + "Tests fail with: ZE_RESULT_ERROR_DEVICE_LOST, code 0x70000001", { + // [Tracking number: E#111369] + ".*OVInferRequestMultithreadingTests.canRun3SyncRequestsConsistently.*" + }); +#endif + + _skipRegistry.addPatterns(backendName.isZero(), "Most ProfilingTest_VPU3700 instances break sporadically, only stable instances are left, #65844", { + ".*precommit_profilingDisabled/ProfilingTest_VPU3700.*", + ".*precommit_profilingDisabled_drv/ProfilingTest_VPU3700.*", + ".*precommit_profilingEnabled_drv/ProfilingTest_VPU3700.*", + }); + + _skipRegistry.addPatterns(backendName.isIMD(), "IMD/Simics do not support the tests", + { + // [Tracking number: E#81065] + ".*smoke_ClassPluginProperties.*DEVICE_UUID.*", + }); + _skipRegistry.addPatterns(backendName.isIMD(), "Run long time on IMD/Simics", + { + // [Tracking number: E#85488] + ".*PreprocessingPrecisionConvertTestNPU.*", + }); + + // [Track number: E#83423] + _skipRegistry.addPatterns(!backendName.isZero() || !devices.has3720(), + "Tests enabled only for L0 NPU3720", { + ".*smoke_VariableStateBasic.*" + }); + + // [Track number: E#83708] + _skipRegistry.addPatterns(backendName.isZero(), + "MemoryLSTMCellTest failing with NOT_IMPLEMENTED", { + ".*smoke_MemoryLSTMCellTest.*" + }); + + // [Track number: E#118999] + _skipRegistry.addPatterns(backendName.isZero(), + "Newly enabled, never tested", { + ".*smoke_BehaviorTests/OVCompiledGraphImportExportTest.importExportedFunctionConstantResultOnly.*", + ".*smoke_BehaviorTests_OVClassImportExportTestP/OVClassCompiledModelImportExportTestP.smoke_ImportNetworkThrowWithDeviceName.*", + ".*ClassExecutableNetworkTestSuite1NPU.PropertyIsSupportedAndImmutableAndGet.*", + ".*ClassExecutableNetworkTestSuite2NPU.PropertyIsSupportedAndImmutableAndCanNotSet.*", + ".*smoke_BehaviorTests_ClassPluginPropertiesTest/ClassPluginPropertiesTestSuite2NPU.CanNotSetImmutableProperty.*", + ".*smoke_BehaviorTests_ClassPluginPropertiesOptsTest1NPU/ClassPluginPropertiesTestSuite3NPU.CanGetPropertyWithOptionsNotAffectingCore.*", + ".*ClassPluginPropertiesTestSuite4NPU.CanNotSetGetInexistentProperty.*", + ".*BehaviorTests_OVCheckSetSupportedRWMandatoryMetricsPropsTests/OVCheckSetSupportedRWMetricsPropsTests.ChangeCorrectProperties.*" + }); + + _skipRegistry.addPatterns(!backendName.isZero() || !devices.has3720(), + "QueryNetwork is only supported by 3720 platform", { + ".*QueryNetworkTestSuite.*" + }); + + _skipRegistry.addPatterns( + devices.count() > 1, + "Some NPU Plugin metrics require single device to work in auto mode or set particular device", + { + ".*OVClassGetConfigTest.*GetConfigNoThrow.*", + ".*OVClassGetConfigTest.*GetConfigHeteroNoThrow.*", + }); + + // [Tracking number: E#111455] + _skipRegistry.addPatterns("Failing properties tests for AUTO / MULTI", { + ".*OVCheckSetSupportedRWMetricsPropsTests.ChangeCorrectProperties.*MULTI.*LOG_LEVEL.*", + ".*OVCheckSetSupportedRWMetricsPropsTests.ChangeCorrectProperties.*AUTO.*LOG_LEVEL.*" + }); + + // [Tracking number: E#99817] + _skipRegistry.addPatterns(backendName.isZero() && devices.has3720(), + "Disabled tests for NPU3720", { + ".*InferRequestVariableStateTest.inferreq_smoke_VariableState_2infers.*", + ".*OVInferRequestIOTensorTest.*InferStaticNetworkSetChangedInputTensorThrow.*" + }); + + // [Tracking number: E#114903] + _skipRegistry.addPatterns(devices.has3720(), + "Tests fail when using latest OV commit from ww09", { + ".*smoke_RandomUniform/RandomLayerTest_NPU3720.SW.*", + }); + + // TODO + _skipRegistry.addPatterns( + "GetExecGraphInfo function is not implemented for NPU plugin", { + ".*CanCreateTwoCompiledModelsAndCheckRuntimeModel.*", + }); + + // TODO + _skipRegistry.addPatterns("Fails with CID", { + ".*smoke_BehaviorTests_OVClassLoadNetworkTest/OVClassLoadNetworkTestNPU.LoadNetworkHETEROWithDeviceIDNoThrow.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns(devices.has3700(), + "error: Value of: ex.what(), Expected: has substring device xml header", { + ".*smoke_OVClassImportExportTestP/OVClassCompiledModelImportExportTestP.smoke_ImportNetworkThrowWithDeviceName/0.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns(devices.has3700(), + "The device candidate list should not include the meta plugin for MULTI", { + ".*OVCheckSetSupportedRWMandatoryMetricsPropsTests/OVCheckSetSupportedRWMetricsPropsTests.ChangeCorrectProperties/target_device=MULTI.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns(devices.has3700(), + "The device candidate list should not include the meta plugin for AUTO", { + ".*OVCheckSetSupportedRWMandatoryMetricsPropsTests/OVCheckSetSupportedRWMetricsPropsTests.ChangeCorrectProperties/target_device=AUTO.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns(devices.has3700(), + "Disabled tests for NPU3700", { + ".*smoke_BehaviorTest/CompileForDifferentPlatformsTests.CompilationForSpecificPlatform.*DEVICE_ID_3720.*", + ".*smoke_BehaviorTests/OVCompiledGraphImportExportTest.importExportedFunctionConstantResultOnly.*", + ".*smoke_BehaviorTests/OVCompiledGraphImportExportTest.importExportedIENetworkConstantResultOnly.*" + }); + +#ifdef WIN32 + // [Track number: CVS-128116] + _skipRegistry.addPatterns("Unicode paths for ov::cache_dir are not correctly handled on Windows", + {".*CompiledKernelsCacheTest.*CanCreateCacheDirAndDumpBinariesUnicodePath.*"}); +#endif + + // [Tracking number: E#108600] + _skipRegistry.addPatterns(backendName.isZero(), + "Unsupported NPU properties", { + ".*OVCheckSetSupportedRWMetricsPropsTests.ChangeCorrectProperties.*EXECUTION_MODE_HINT.*", + ".*OVCheckChangePropComplieModleGetPropTests_InferencePrecision.*", + ".*OVCheckMetricsPropsTests_ModelDependceProps.*", + ".*OVCheckChangePropComplieModleGetPropTests_DEVICE_ID.*ENABLE_CPU_PINNING.*", + ".*OVClassCompileModelAndCheckSecondaryPropertiesTest.*" + }); + + // [Tracking number: E#108600] + _skipRegistry.addPatterns(backendName.isZero(), + "Failing properties tests", { + ".*OVSpecificDeviceSetConfigTest.GetConfigSpecificDeviceNoThrow.*", + ".*OVPropertiesIncorrectTests.SetPropertiesWithIncorrectKey.*DEVICE_ID.*", + ".*OVPropertiesIncorrectTests.SetPropertiesWithIncorrectKey.*NPU_COMPILATION_MODE.*", + ".*OVPropertiesIncorrectTests.SetPropertiesWithIncorrectKey.*NPU_COMPILATION_MODE_PARAMS.*", + ".*OVPropertiesIncorrectTests.SetPropertiesWithIncorrectKey.*NPU_PROFILING_OUTPUT_FILE.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns("Disabled all tests CompileForDifferentPlatformsTests with config NPU_COMPILER_TYPE_DRIVER", { + ".*smoke_BehaviorTest/CompileForDifferentPlatformsTests.*" + }); + + // TODO + _skipRegistry.addPatterns( + "GetExecGraphInfo function is not implemented for NPU plugin", { + ".*CanCreateTwoCompiledModelsAndCheckRuntimeModel.*", + }); + + // TODO + _skipRegistry.addPatterns("Fails with CID", { + ".*smoke_BehaviorTests_OVClassLoadNetworkTest/OVClassLoadNetworkTestNPU.LoadNetworkHETEROWithDeviceIDNoThrow.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns(devices.has3700(), + "error: Value of: ex.what(), Expected: has substring device xml header", { + ".*smoke_OVClassImportExportTestP/OVClassCompiledModelImportExportTestP.smoke_ImportNetworkThrowWithDeviceName/0.*" + }); + + // [Tracking number: E#114623] + _skipRegistry.addPatterns(!devices.has3720(), + "The private platform names cannot be identified via the \"ov::available_devices\" configuration.", { + ".*smoke_BehaviorTests_OVClassSetDefaultDeviceIDPropTest/OVClassSetDefaultDeviceIDPropTest.SetDefaultDeviceIDNoThrow.*", + ".*smoke_BehaviorTests_OVClassSpecificDeviceTest/OVSpecificDeviceGetConfigTest.GetConfigSpecificDeviceNoThrow.*", + ".*smoke_BehaviorTests_OVClassSpecificDeviceTest/OVSpecificDeviceTestSetConfig.SetConfigSpecificDeviceNoThrow.*" + }); + + // [Tracking number: E#114624] + _skipRegistry.addPatterns( + "The tests are not actually running the compiler-in-driver module.", { + ".*smoke_BehaviorTests_OVCheckSetSupportedRWMetricsPropsTests.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns(devices.has3700(), + "The device candidate list should not include the meta plugin for MULTI", { + ".*OVCheckSetSupportedRWMandatoryMetricsPropsTests/OVCheckSetSupportedRWMetricsPropsTests.ChangeCorrectProperties/target_device=MULTI.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns(devices.has3700(), + "The device candidate list should not include the meta plugin for AUTO", { + ".*OVCheckSetSupportedRWMandatoryMetricsPropsTests/OVCheckSetSupportedRWMetricsPropsTests.ChangeCorrectProperties/target_device=AUTO.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns(devices.has3700(), + "Disabled tests for NPU3700", { + ".*smoke_BehaviorTest/CompileForDifferentPlatformsTests.CompilationForSpecificPlatform.*DEVICE_ID_3720.*", + ".*smoke_BehaviorTests/OVCompiledGraphImportExportTest.importExportedFunctionConstantResultOnly.*", + ".*smoke_BehaviorTests/OVCompiledGraphImportExportTest.importExportedIENetworkConstantResultOnly.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns(devices.has3700() || devices.has3720(), + "Disabled tests for NPU3720", { + ".*smoke.*_BehaviorTests/OVInferRequestCheckTensorPrecision.*type=i16.*", + ".*smoke.*_BehaviorTests/OVInferRequestCheckTensorPrecision.*type=u16.*", + ".*smoke.*_BehaviorTests/OVInferRequestCheckTensorPrecision.*type=u64.*", + ".*smoke_OVClassLoadNetworkTest/OVClassLoadNetworkTestNPU.*", + }); + + // [Tracking number: E#112064] + _skipRegistry.addPatterns(backendName.isZero(), + "Failing core threading tests", { + ".*CoreThreadingTest.smoke_QueryModel.*", + ".*CoreThreadingTestsWithIter.smoke_CompileModel.*", + ".*CoreThreadingTestsWithIter.smoke_CompileModel_Accuracy_SingleCore.*", + ".*CoreThreadingTestsWithIter.smoke_CompileModel_Accuracy_MultipleCores.*", + ".*CoreThreadingTestsWithIter.nightly_AsyncInfer_ShareInput.*" + }); + + // [Tracking number: E#108600] + _skipRegistry.addPatterns(backendName.isZero(), + "Failing properties tests", { + ".*OVSpecificDeviceSetConfigTest.GetConfigSpecificDeviceNoThrow.*", + ".*OVPropertiesIncorrectTests.SetPropertiesWithIncorrectKey.*DEVICE_ID.*", + ".*OVPropertiesIncorrectTests.SetPropertiesWithIncorrectKey.*NPU_COMPILATION_MODE.*", + ".*OVPropertiesIncorrectTests.SetPropertiesWithIncorrectKey.*NPU_COMPILATION_MODE_PARAMS.*", + ".*OVPropertiesIncorrectTests.SetPropertiesWithIncorrectKey.*NPU_PROFILING_OUTPUT_FILE.*" + }); + + // [Tracking number: E#117582] + _skipRegistry.addPatterns(backendName.isZero(), + "Failing core threading test with cache enabled", { + ".*CoreThreadingTest.*CoreThreadingTestsWithCacheEnabled.*" + }); + + // [Tracking number: E#118331] + _skipRegistry.addPatterns(backendName.isZero() && !devices.has3720(), + "platform and compiler_type are private", { + ".*smoke_Multi_BehaviorTests/OVInferRequestCallbackTests.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestCallbackTests.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestCallbackTestsNPU.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestCallbackTestsNPU.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestIOTensorTestNPU.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestIOTensorTest.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestMultithreadingTests.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestMultithreadingTestsNPU.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestPerfCountersExceptionTest.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestPerfCountersTest.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestWaitTests.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestMultithreadingTests.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestMultithreadingTestsNPU.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestPerfCountersExceptionTest.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestPerfCountersTest.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestWaitTests.*", + ".*smoke_OVClassNetworkTestP/OVClassNetworkTestPNPU.*", + ".*smoke_OVClassLoadNetworkTest/OVClassLoadNetworkTestNPU.*", + ".*smoke_Hetero_BehaviorTests_VariableState/OVInferRequestVariableStateTest.*" + }); + + // [Tracking number: E#118331] + _skipRegistry.addPatterns(backendName.isZero(), + "Private properties cannot be accessed by HETERO compiled model", { + ".*smoke_Hetero_BehaviorTests.*OVClassCompiledModelGetPropertyTest_MODEL_PRIORITY.*", + ".*smoke_Hetero_BehaviorTests.*OVClassCompiledModelGetPropertyTest_EXEC_DEVICES.*", + ".*smoke_Hetero_BehaviorTests.*OVCompileModelGetExecutionDeviceTests.*" + }); + + // [Tracking number: E#118331] + _skipRegistry.addPatterns(backendName.isZero() && !devices.has3720(), + "platform and compiler_type are private", { + ".*smoke_Multi_BehaviorTests/OVInferRequestCallbackTests.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestCallbackTests.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestCallbackTestsNPU.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestCallbackTestsNPU.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestIOTensorTestNPU.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestIOTensorTest.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestMultithreadingTests.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestPerfCountersTest.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestPerfCountersExceptionTest.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestWaitTests.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestMultithreadingTestsNPU.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestPerfCountersTest.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestPerfCountersExceptionTest.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestWaitTests.*", + ".*smoke_OVClassNetworkTestP/OVClassNetworkTestPNPU.*" + }); + + // [Tracking number: E#118348] + _skipRegistry.addPatterns(devices.has3700() && backendName.isZero(), + "Failing infer request tests on dKMB using CID", { + ".*smoke_BehaviorTests/OVInferRequestPerfCountersExceptionTest.perfCountWereNotEnabledExceptionTest.*", + ".*smoke_Auto_BehaviorTests/OVInferRequestPerfCountersExceptionTest.perfCountWereNotEnabledExceptionTest.*", + ".*smoke_Multi_BehaviorTests/OVInferRequestPerfCountersExceptionTest.perfCountWereNotEnabledExceptionTest.*", + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_QueryState.*", + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_SetState.*", + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_Reset.*", + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_2infers_set.*", + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_2infers.*", + ".*smoke_Hetero_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_QueryState.*", + ".*smoke_Hetero_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_SetState.*", + ".*smoke_Hetero_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_Reset.*", + ".*smoke_Hetero_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_2infers_set.*", + ".*smoke_Hetero_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_2infers.*" + }); + + // [Tracking number: E#125086] + _skipRegistry.addPatterns(devices.has3720() && backendName.isZero(), + "Failing tests after functional tests migration to OV", { + #ifdef WIN32 + ".*OVInferRequestPerfCountersExceptionTest.perfCountWereNotEnabledExceptionTest.*", + #elif defined(__linux__) + ".*OVInferRequestMultithreadingTests.canRun3AsyncRequestsConsistently.*", + #endif + ".*OVCompiledModelPropertiesDefaultSupportedTests.CanCompileWithDefaultValueFromPlugin.*" + }); + + // [Tracking number: E#118348] + _skipRegistry.addPatterns(devices.has3700() && backendName.isZero(), + "Failing infer request tests on dKMB using MLIR", { + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_QueryState.*", + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_SetState.*", + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_Reset.*", + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_2infers_set.*", + ".*smoke_BehaviorTests_VariableState/OVInferRequestVariableStateTest.inferreq_smoke_VariableState_2infers.*" + }); + + // [Tracking number: E#116494] + _skipRegistry.addPatterns( + "NPU plugin doesn't implement `set_tensors` function", { + ".*OVInferRequestBatchedTests.SetInputTensorsBase.*", + ".*OVInferRequestBatchedTests.SetInputTensorsAsync.*", + ".*OVInferRequestBatchedTests.SetInputTensors_override_with_set.*", + ".*OVInferRequestBatchedTests.SetInputTensorsBase_Caching.*", + ".*OVInferRequestBatchedTests.SetInputTensors_Multiple_Infer.*", + ".*OVInferRequestBatchedTests.SetInputTensors_Can_Infer_Dynamic.*", + ".*OVInferRequestBatchedTests.SetInputTensors_Get_Tensor_Not_Allowed.*", + ".*OVInferRequestBatchedTests.SetInputTensors_Correct_all.*", + ".*OVInferRequestBatchedTests.SetInputTensors_Cache_CheckDeepCopy.*" + }); + + // [Tracking number: E#118381] + _skipRegistry.addPatterns("Comparation is failed, SLT need to be updated.", { + ".*smoke.*GridSample_Tiling/GridSampleLayerTest.*align_corners=0.*Mode=nearest_padding_mode=zeros.*", + ".*smoke.*GridSample_Tiling/GridSampleLayerTest.*align_corners=0.*Mode=nearest_padding_mode=border.*", + ".*smoke.*GridSample_Tiling/GridSampleLayerTest.*align_corners=0.*Mode=nearest_padding_mode=reflection.*" + }); + + // [Tracking number: E#116575] + _skipRegistry.addPatterns( + "NPU fails for `OVIterationChaining.Simple` tests", { + ".*OVIterationChaining.Simple.*" + }); + + // [Tracking number: E#116596] + _skipRegistry.addPatterns( + "Missing model ops in profiling info", { + ".*OVInferRequestPerfCountersTest.CheckOperationInProfilingInfo.*" + }); + + // [Tracking number: E#118045] + _skipRegistry.addPatterns( + "NPU needs to implement ROITensor logic in zero_infer_request", { + ".*OVInferRequestInferenceTests.Inference_ROI_Tensor/roi_nchw.*" + }); + + // [Tracking number: E#116761] + _skipRegistry.addPatterns("OVClassQueryModel tests do not work with COMPILER_TYPE=DRIVER", { + ".*OVClassQueryModelTest.QueryModelHETEROWithDeviceIDNoThrow.*", + ".*OVClassQueryModelTest.QueryModelWithBigDeviceIDThrows.*", + ".*OVClassQueryModelTest.QueryModelWithInvalidDeviceIDThrows.*" + }); + + // [Tracking number: E#116762] + _skipRegistry.addPatterns("softMaxDynamicTest4D_NPU3720 tests do not work with COMPILER_TYPE=DRIVER", { + ".*SoftMaxLayerTestNPU.CompareWithRef.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns("CheckWrongGraphExtAndThrow tests do not work with COMPILER_TYPE=DRIVER", { + ".*DriverCompilerAdapterExpectedThrowNPU.CheckWrongGraphExtAndThrow.*" + }); + + // [Tracking number: E#109040] + _skipRegistry.addPatterns("Skip tests that can not wrong when DRIVER is default compiler type", { + ".*OVClassLoadNetworkTestNPU.LoadNetworkHETEROWithDeviceIDNoThrow.*", + ".*MatMulTransposeConcatTest.*" + }); + + // [Tracking number: E#121043] + _skipRegistry.addPatterns("Compiler adapter is not extracting network name from metadata", { + ".*smoke_BehaviorTests/OVClassCompiledModelGetPropertyTest.GetMetricNoThrow_NETWORK_NAME.*" + }); + + // [Tracking number: E#121347] + _skipRegistry.addPatterns("Error message for empty model from stream must be changed to have \"device xml header\"", { + ".*smoke_BehaviorTests/OVClassCompiledModelImportExportTestP.smoke_ImportNetworkThrowWithDeviceName.*", + ".*smoke_Hetero_BehaviorTests/OVClassCompiledModelImportExportTestP.smoke_ImportNetworkThrowWithDeviceName.*" + }); + + // [Tracking number: E#121448] + _skipRegistry.addPatterns(devices.has3700() && backendName.isZero(), "CID from driver 31.0.100.1937 doesn't support certain properties", { + ".*smoke_BehaviorTests/OVClassCompiledModelPropertiesTests.canCompileModelWithPropertiesAndCheckGetProperty.*NPU_DYNAMIC_SHAPE_TO_STATIC.*", + ".*smoke_BehaviorTests/OVClassCompileModelWithCorrectPropertiesTest.CompileModelWithCorrectPropertiesTest.*NPU_DYNAMIC_SHAPE_TO_STATIC.*", + ".*smoke_BehaviorTests/OVClassCompiledModelGetPropertyTest_MODEL_PRIORITY.GetMetricNoThrow.*NPU_DYNAMIC_SHAPE_TO_STATIC.*", + ".*smoke_BehaviorTests/OVClassCompiledModelGetPropertyTest_MODEL_PRIORITY.GetMetricNoThrow.*NPU_DYNAMIC_SHAPE_TO_STATIC.*", + ".*smoke_BehaviorTests/OVClassCompiledModelGetPropertyTest_EXEC_DEVICES.CanGetExecutionDeviceInfo.*NPU_DYNAMIC_SHAPE_TO_STATIC.*", + ".*smoke_BehaviorTests/OVCompileModelGetExecutionDeviceTests.CanGetExecutionDeviceInfo.*NPU_DYNAMIC_SHAPE_TO_STATIC.*", + // COMPILATION_NUM_THREADS + ".*smoke_BehaviorTests/OVCompiledModelPropertiesDefaultSupportedTests.CanCompileWithDefaultValueFromPlugin.*" + }); + + _skipRegistry.addPatterns("NPU cannot set properties for compiled models", { + ".*OVClassCompiledModelSetCorrectConfigTest.canSetConfig.*" + }); + + // [Tracking number: CVS-139118] + _skipRegistry.addPatterns("Failing runtime model tests", { + ".*OVCompiledModelGraphUniqueNodeNamesTest.CheckUniqueNodeNames.*", + ".*OVExecGraphSerializationTest.ExecutionGraph.*" + }); + + return _skipRegistry; + }(); + // clang-format on + + std::vector matchingPatterns; + const auto currentTestName = getCurrentTestName(); + matchingPatterns.emplace_back(skipRegistry.getMatchingPattern(currentTestName)); + + return matchingPatterns; +}