natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Support inference_precision::undefined in CPU and use it in ExecutionMode::ACCURACY (#24118) 

### Details:
- *Support inference_precision::undefined in CPU and use it in
ExecutionMode::ACCURACY*

### Tickets:
 - *136167*
This commit is contained in:
Nikolay Shchegolev 2024-05-17 15:57:37 +04:00 committed by GitHub
parent 3f3c86ddad
commit 88b0309830
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 266 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/plugins/intel_cpu/src/config.cpp
View File

@ -289,8 +289,8 @@ void Config::readProperties(const ov::AnyMap& prop, const ModelType modelType) {
if (hasHardwareSupport(ov::element::f16)) {
inferencePrecision = ov::element::f16;
}
} else if (prec == ov::element::f32) {
inferencePrecision = ov::element::f32;
} else if (one_of(prec, element::f32, element::undefined)) {
inferencePrecision = prec;
} else {
OPENVINO_THROW("invalid value");
}
@ -299,7 +299,7 @@ void Config::readProperties(const ov::AnyMap& prop, const ModelType modelType) {
val.as<std::string>(),
" for property key ",
ov::hint::inference_precision.name(),
". Supported values: bf16, f16, f32");
". Supported values: bf16, f16, f32, undefined");
}
} else if (ov::intel_cpu::cpu_runtime_cache_capacity.name() == key) {
int val_i = -1;
@ -387,7 +387,7 @@ void Config::readProperties(const ov::AnyMap& prop, const ModelType modelType) {
if (mayiuse(avx512_core_bf16))
inferencePrecision = ov::element::bf16;
} else {
inferencePrecision = ov::element::f32;
inferencePrecision = ov::element::undefined;
}
}

2
src/plugins/intel_cpu/src/graph.cpp
View File

@ -1735,7 +1735,7 @@ void Graph::EnforceInferencePrecision() {
const auto inferPrec = getConfig().inferencePrecision;
if (inferPrec == ov::element::f32)
if (one_of(inferPrec, element::f32, element::undefined))
return; // nothing to do, only precision reduction is currently allowed
#if defined(OPENVINO_ARCH_ARM) || defined(OPENVINO_ARCH_ARM64)
if (inferPrec == ov::element::f16)

22
src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp
View File

@ -344,11 +344,13 @@ void Transformations::PreLpt(const std::vector<ov::element::Type>& defaultPrecis
if (!hasHardwareSupport(ov::element::bf16))
map.insert({ov::element::bf16, ov::element::f32});
#if defined(OPENVINO_ARCH_ARM) || defined(OPENVINO_ARCH_ARM64)
if (inferencePrecision != ov::element::f16) {
map.insert({ov::element::f16, ov::element::f32});
if (!one_of(inferencePrecision, element::f16, element::undefined)) {
map.insert({element::f16, element::f32});
}
#else
map.insert({ov::element::f16, ov::element::f32});
if (inferencePrecision != element::undefined) {
map.insert({element::f16, element::f32});
}
#endif
return map;
};
@ -371,7 +373,7 @@ void Transformations::PreLpt(const std::vector<ov::element::Type>& defaultPrecis
false);
}
#else
static const auto precisions = get_convert_precisions();
const auto precisions = get_convert_precisions();
#endif
CPU_REGISTER_PASS_COMMON(manager, ov::pass::KeepConstAndDecompression);
CPU_SET_CALLBACK_COMMON(manager,
@ -511,10 +513,10 @@ void Transformations::PreLpt(const std::vector<ov::element::Type>& defaultPrecis
// 2. GroupNormalizationDecomposition produce MVN, and MVN have a conditional pass MVN6Decomposition. If call MVN6Decomposition again after
// snippets pipeline as well, where MVN is decomposed to simple ops, these simple ops will not tokenized into subgraph again.
// CVS-134277 to fully enable GN as snippets to disable this GroupNormalizationDecomposition entirly.
if (node->is_dynamic() || inferencePrecision != element::f32)
if (node->is_dynamic() || !one_of(inferencePrecision, element::f32, element::undefined))
return false;
const auto group_norm = ov::as_type_ptr<const ov::op::v12::GroupNormalization>(node);
if (!group_norm)
if (!group_norm || !implication(inferencePrecision == element::undefined, group_norm->get_element_type() == element::f32))
return false;
const auto num_groups = static_cast<size_t>(group_norm->get_num_groups());
const auto shape = group_norm->get_input_partial_shape(0).to_shape();
@ -809,7 +811,7 @@ void Transformations::MainSnippets(void) {
// - CPU Node Subgraph requires bf16 on output when inference precision is bf16.
// To avoid sitations when Transpose is not alone node between MatMul and Result,
// Plugin disables Transpose tokenization on output
bool mha_token_enable_transpose_on_output = (inferencePrecision == ov::element::f32);
bool mha_token_enable_transpose_on_output = one_of(inferencePrecision, element::f32, element::undefined);
size_t concurrency = config.streamExecutorConfig.get_threads_per_stream();
if (concurrency == 0)
concurrency = parallel_get_max_threads();
@ -836,7 +838,7 @@ void Transformations::MainSnippets(void) {
#if defined(OPENVINO_ARCH_ARM64)
CPU_REGISTER_PASS_ARM(snippetsManager, SnippetsMarkSkipped);
#else
CPU_REGISTER_PASS_X64(snippetsManager, SnippetsMarkSkipped, inferencePrecision != ov::element::f32);
CPU_REGISTER_PASS_X64(snippetsManager, SnippetsMarkSkipped, inferencePrecision == ov::element::bf16);
#endif
}
CPU_REGISTER_PASS_X64(snippetsManager, snippets::pass::SnippetsTokenization, tokenization_config);
@ -855,7 +857,7 @@ void Transformations::MainSnippets(void) {
false;
#else
dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core) &&
one_of(inferencePrecision, ov::element::bf16, ov::element::f32);
one_of(inferencePrecision, ov::element::bf16, ov::element::f32, element::undefined);
#endif
if (!isMHASupported) {
CPU_DISABLE_PASS_COMMON(snippetsManager, snippets::pass::TokenizeMHASnippets);
@ -869,7 +871,7 @@ void Transformations::MainSnippets(void) {
return false;
const auto in_type0 = matmul->get_input_element_type(0);
const auto in_type1 = matmul->get_input_element_type(1);
if (in_type0 == ov::element::f32 && in_type1 == ov::element::f32 && inferencePrecision == ov::element::f32)
if (in_type0 == ov::element::f32 && in_type1 == ov::element::f32 && one_of(inferencePrecision, element::f32, element::undefined))
return true;
// [114487] brgemm kernel in oneDNN requires brgemm_copy_b kernel if MatMul node has transposed_b=True
// The current solution with ExtractExplicitMatMulTranspose pass is slower for non-f32 cases than using of brgemm_copy_b kernel

2
src/plugins/intel_cpu/tests/functional/custom/behavior/ov_plugin/properties.cpp
View File

@ -256,7 +256,7 @@ const std::map<ov::hint::ExecutionMode, ExpectedModeAndType> expectedTypeByMode
{ov::hint::ExecutionMode::PERFORMANCE, {ov::hint::ExecutionMode::PERFORMANCE,
expected_precision_for_performance_mode}},
{ov::hint::ExecutionMode::ACCURACY, {ov::hint::ExecutionMode::ACCURACY,
ov::element::f32}},
ov::element::undefined}},
};
TEST_F(OVClassConfigTestCPU, smoke_PluginSetConfigExecutionModeExpectCorrespondingInferencePrecision) {

34
src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/include/undefined_et.hpp Normal file
View File

@ -0,0 +1,34 @@
// Copyright (C) 2018-2024 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "utils/cpu_test_utils.hpp"
#include "shared_test_classes/base/ov_subgraph.hpp"
namespace ov {
namespace test {
typedef std::tuple<
element::Type_t, // Data element type
ov::AnyMap // Additional configuration
> UndefinedEtCpuParams;
class UndefinedEtSubgraphTest : public testing::WithParamInterface<UndefinedEtCpuParams>,
public CPUTestUtils::CPUTestsBase,
virtual public SubgraphBaseStaticTest {
public:
static std::string getTestCaseName(const testing::TestParamInfo<UndefinedEtCpuParams>& obj);
protected:
void SetUp() override;
void generate_inputs(const std::vector<ov::Shape>& target_shapes) override;
hint::ExecutionMode m_mode;
element::Type m_data_et = element::undefined;
};
} // namespace test
} // namespace ov

169
src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/classes/undefined_et.cpp Normal file
View File

@ -0,0 +1,169 @@
// Copyright (C) 2018-2024 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
// Modes: ACCURACY; PERFORMANCE + INFERENCE_PRECISION->undefined
// Expected: original execution type from the IR.
//
// -----------
// | Parameter |
// -----------
// | f32/f16/bf16
// -----------------
// | RandomUniform | supports execution in f32/f16/bf16 on x86 and ARM
// -----------------
// |
// -----------
// | Convert |
// -----------
// | f32
// -----------
// | Eltwise |
// -----------
// |
// -----------
// | Result |
// -----------
#include "custom/subgraph_tests/include/undefined_et.hpp"
namespace ov {
namespace test {
std::string UndefinedEtSubgraphTest::getTestCaseName(const testing::TestParamInfo<UndefinedEtCpuParams>& obj) {
std::ostringstream result;
result << "DataET=" << std::get<0>(obj.param);
const auto& config = std::get<1>(obj.param);
if (!config.empty()) {
result << "_PluginConf={";
for (const auto& conf_item : config) {
result << "_" << conf_item.first << "=";
conf_item.second.print(result);
result << "_";
}
result << "}";
}
return result.str();
}
void UndefinedEtSubgraphTest::SetUp() {
targetDevice = test::utils::DEVICE_CPU;
const auto& params = this->GetParam();
m_data_et = std::get<0>(params);
const auto& config = std::get<1>(params);
configuration.insert(config.begin(), config.end());
auto it = configuration.find(hint::execution_mode.name());
ASSERT_NE(configuration.end(), it);
m_mode = it->second.as<hint::ExecutionMode>();
init_input_shapes({ {{}, {{3}}}, {{}, {{1}}}, {{}, {{1}}} });
auto param_0 = std::make_shared<op::v0::Parameter>(element::i64, inputDynamicShapes[0]);
param_0->set_friendly_name("shape");
auto param_1 = std::make_shared<op::v0::Parameter>(m_data_et, inputDynamicShapes[1]);
param_1->set_friendly_name("minval");
auto param_2 = std::make_shared<op::v0::Parameter>(m_data_et, inputDynamicShapes[2]);
param_2->set_friendly_name("maxval");
auto rnd_unfm = std::make_shared<op::v8::RandomUniform>(param_0, param_1, param_2, m_data_et);
auto cvt_f32 = std::make_shared<op::v0::Convert>(rnd_unfm, element::f32);
auto logical_not = std::make_shared<op::v1::LogicalNot>(cvt_f32);
function = std::make_shared<ov::Model>(OutputVector{logical_not->output(0)}, ParameterVector{param_0, param_1, param_2}, "UndefinedET");
}
template<typename TD, typename TS>
void fill_data(TD* dst, const TS* src, size_t len) {
for (size_t i = 0llu; i < len; i++) {
dst[i] = static_cast<TD>(src[i]);
}
}
void UndefinedEtSubgraphTest::generate_inputs(const std::vector<ov::Shape>& target_shapes) {
inputs.clear();
const auto& func_inputs = function->inputs();
#define ET_CASE(P, S, L) \
case P : \
fill_data(tensor->data<ov::element_type_traits<P>::value_type>(), S, L); break;
for (size_t i = 0lu; i < func_inputs.size(); i++) {
const auto& param = func_inputs[i];
const auto& name = param.get_node()->get_friendly_name();
const auto& in_prc = param.get_element_type();
std::shared_ptr<ov::Tensor> tensor;
if (name == "shape") {
static const int64_t shape[] = {3, 5, 7};
tensor = std::make_shared<ov::Tensor>(in_prc, Shape{3});
switch (in_prc) {
ET_CASE(element::i32, shape, 3)
ET_CASE(element::i64, shape, 3)
default:
OPENVINO_THROW("RandomUniform does not support precision ", in_prc, " for the Shape input.");
}
} else if (name == "minval") {
static const float min_val = 0.f;
tensor = std::make_shared<ov::Tensor>(in_prc, target_shapes[i]);
switch (in_prc) {
ET_CASE(ElementType::f32, &min_val, 1)
ET_CASE(ElementType::f16, &min_val, 1)
ET_CASE(ElementType::bf16, &min_val, 1)
default:
OPENVINO_THROW("RandomUniform does not support precision ", in_prc, " for the Minval input.");
}
} else if (name == "maxval") {
static const float max_val = 20.f;
tensor = std::make_shared<ov::Tensor>(in_prc, target_shapes[i]);
switch (in_prc) {
ET_CASE(ElementType::f32, &max_val, 1)
ET_CASE(ElementType::f16, &max_val, 1)
ET_CASE(ElementType::bf16, &max_val, 1)
default:
OPENVINO_THROW("RandomUniform does not support precision ", in_prc, " for the Maxval input.");
}
}
#undef ET_CASE
inputs.insert({param.get_node_shared_ptr(), *tensor});
}
}
TEST_P(UndefinedEtSubgraphTest, CompareWithRefs) {
run();
if (IsSkipped()) {
return;
}
ASSERT_EQ(compiledModel.get_property(ov::hint::execution_mode), m_mode);
ASSERT_EQ(compiledModel.get_property(ov::hint::inference_precision), element::undefined);
size_t rnd_unfm_counter = 0lu;
size_t logical_not_counter = 0lu;
for (const auto& node : compiledModel.get_runtime_model()->get_ops()) {
auto rt_info = node->get_rt_info();
auto it = rt_info.find(exec_model_info::LAYER_TYPE);
ASSERT_NE(rt_info.end(), it);
auto op_name = it->second.as<std::string>();
if (op_name == "RandomUniform") {
ASSERT_EQ(node->get_output_element_type(0), m_data_et);
rnd_unfm_counter++;
}
if (op_name == "Eltwise") {
ASSERT_EQ(node->get_output_element_type(0), element::f32);
logical_not_counter++;
}
}
ASSERT_EQ(rnd_unfm_counter, 1lu);
ASSERT_EQ(logical_not_counter, 1lu);
};
} // namespace test
} // namespace ov

29
src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/common/undefined_et.cpp Normal file
View File

@ -0,0 +1,29 @@
// Copyright (C) 2018-2024 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
#include "custom/subgraph_tests/include/undefined_et.hpp"
namespace ov {
namespace test {
namespace {
static const std::vector<ElementType> data_et = {
element::f32,
element::f16,
element::bf16
};
static const std::vector<ov::AnyMap> plugin_config{
{{hint::execution_mode.name(), hint::ExecutionMode::ACCURACY}},
{{hint::execution_mode.name(), hint::ExecutionMode::PERFORMANCE}, {hint::inference_precision.name(), element::undefined}}
};
INSTANTIATE_TEST_SUITE_P(smoke_, UndefinedEtSubgraphTest,
::testing::Combine(
::testing::ValuesIn(data_et),
::testing::ValuesIn(plugin_config)),
UndefinedEtSubgraphTest::getTestCaseName);
} // namespace
} // namespace test
} // namespace ov

24
src/plugins/intel_cpu/tests/functional/shared_tests_instances/core_config.cpp
View File

@ -8,18 +8,26 @@ namespace ov {
namespace test {
void core_configuration(ov::test::SubgraphBaseTest* test) {
//force fp32 inference precision if it is not configured specially
if (!test->configuration.count(ov::hint::inference_precision.name())) {
test->configuration.insert({ov::hint::inference_precision.name(), ov::element::f32.to_string()});
}
auto& config = test->configuration;
auto exec_mode = config.find(hint::execution_mode.name());
auto inf_prc = config.find(hint::inference_precision.name());
// todo: issue: 123320
// Force fp32 inference precision if it is not configured specially
if (inf_prc == config.end() &&
(exec_mode == config.end() || exec_mode->second != hint::ExecutionMode::ACCURACY)) {
config.insert({hint::inference_precision.name(), element::f32.to_string()});
}
// todo: issue: 123320
if (!((inf_prc != config.end() && inf_prc->second == element::undefined)
|| (inf_prc == config.end() && exec_mode != config.end() && exec_mode->second == hint::ExecutionMode::ACCURACY))) {
test->convert_precisions.insert({ov::element::bf16, ov::element::f32});
test->convert_precisions.insert({ov::element::f16, ov::element::f32});
}
// Enable CPU pinning in CPU funtional tests to save validation time of Intel CPU plugin func tests (parallel)
// on Windows
test->configuration.insert({ov::hint::enable_cpu_pinning.name(), true});
// Enable CPU pinning in CPU funtional tests to save validation time of Intel CPU plugin func tests (parallel)
// on Windows
config.insert({ov::hint::enable_cpu_pinning.name(), true});
}
} // namespace test