natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] fp16 weights decompression works with conversion to f32 (#20427)

This commit is contained in:
Vladislav Golubev 2023-10-17 10:47:15 +02:00 committed by GitHub
parent 05297a5572
commit 93a70497bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 47 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp
View File

@ -225,12 +225,18 @@ void Transformations::PreLpt(const std::vector<ov::element::Type>& defaultPrecis
return false;
}
// TODO: Uncomment when group decompression is supported
// else if (ov::is_type<ov::opset1::Reshape>(consumer)) {
// if (ov::is_type<ov::opset1::Reshape>(consumer)) {
// consumer = get_single_consumer(consumer);
// if (consumer != nullptr && ov::is_type<ov::opset1::MatMul>(consumer)) {
// return false;
// }
// }
if (ov::is_type<ov::opset1::Convert>(consumer)) {
consumer = get_single_consumer(consumer);
if (consumer != nullptr && ov::is_type<ov::opset1::MatMul>(consumer)) {
return false;
}
}
return true;
}, ov::pass::MarkDequantizationSubgraph);
}

53
src/plugins/intel_cpu/tests/functional/subgraph_tests/src/matmul_weights_decompression.cpp
View File

@ -14,18 +14,26 @@ using namespace ov::test;
namespace SubgraphTestsDefinitions {
/*
* Subtract_const(U8/NF4)
* WP - weights precision
* DP - decompression precision
* IP - input precision
* Opt - optional
* Subtract_const(WP)
* /
* Weights(U8/NF4) Convert(F32)
* Weights(WP) Convert(DP)
* | /
* Convert(F32) Reshape
* \ / Multiply_const(F32)
* Subtract(opt) /
* \ Reshape
* \ /
* Multiply
* Convert(DP) Reshape (Opt)
* \ / Multiply_const(DP)
* Subtract(Opt) /
* \ Reshape (Opt)
* \ /
* Multiply
* |
* Data(F32) Transpose(opt)
* Reshape (in case of group decompression)
* |
* Convert (if IP != DP)
* |
* Data(IP) Transpose(Opt)
* \ /
* Matmul
* |
@ -46,6 +54,7 @@ struct ShapeParams {
};
using MatmulWeightsDecompressionParams = std::tuple<ShapeParams,
ov::test::ElementType, // weights precision
ov::test::ElementType, // decompression precision
bool, // transpose on weights
bool, // decompression subtract
bool, // reshape on decompression constants
@ -60,6 +69,7 @@ public:
static std::string getTestCaseName(testing::TestParamInfo<MatmulWeightsDecompressionParams> obj) {
ShapeParams shape_params;
ov::test::ElementType weights_precision;
ov::test::ElementType decompression_precision;
bool transpose;
bool decompression_sub;
bool reshape_on_decompression;
@ -69,6 +79,7 @@ public:
std::tie(shape_params,
weights_precision,
decompression_precision,
transpose,
decompression_sub,
reshape_on_decompression,
@ -81,6 +92,7 @@ public:
result << "weights_shape=" << shape_params.weights_shape << "_";
result << "group_size=" << shape_params.weights_group_size << "_";
result << "weights_precision=" << weights_precision << "_";
result << "decompression_precision=" << decompression_precision << "_";
result << "transpose_weights=" << transpose << "_";
result << "decompression_subtract=" << decompression_sub << "_";
result << "reshape_on_decompression=" << reshape_on_decompression << "_";
@ -100,6 +112,7 @@ protected:
const int group_size,
const ov::element::Type data_precision,
const ov::element::Type weights_precision,
const ov::element::Type decompression_precision,
const bool transpose_weights,
const bool add_subtract,
const bool reshape_on_decompression_constant) {
@ -131,7 +144,7 @@ protected:
}
auto weights = ngraph::builder::makeConstant<uint8_t>(weights_precision, transformed_weights_shape, {}, true);
weights->set_friendly_name("Compressed_weights");
auto weights_convert = std::make_shared<ngraph::opset1::Convert>(weights, data_precision);
auto weights_convert = std::make_shared<ngraph::opset1::Convert>(weights, decompression_precision);
std::shared_ptr<ov::Node> mul_parent = weights_convert;
auto output_channels = *weights_shape.rbegin();
@ -152,7 +165,7 @@ protected:
scaleshift_const_shape.erase(std::remove(scaleshift_const_shape.begin(), scaleshift_const_shape.end(), 1), scaleshift_const_shape.end());
if (add_subtract) {
auto shift_const = ngraph::builder::makeConstant<uint8_t>(weights_precision, scaleshift_const_shape, {}, true);
std::shared_ptr<ov::Node> shift_convert = std::make_shared<ngraph::opset1::Convert>(shift_const, data_precision);
std::shared_ptr<ov::Node> shift_convert = std::make_shared<ngraph::opset1::Convert>(shift_const, decompression_precision);
if (reshape_on_decompression_constant) {
auto shift_reshape_const = ov::opset10::Constant::create(ov::element::i32, {scaleshift_target_shape.size()}, scaleshift_target_shape);
auto shift_reshape = std::make_shared<ov::opset10::Reshape>(shift_convert, shift_reshape_const, false);
@ -161,7 +174,7 @@ protected:
mul_parent = std::make_shared<ov::opset10::Subtract>(weights_convert, shift_convert);
}
std::shared_ptr<ov::Node> scale_const = ngraph::builder::makeConstant<float>(data_precision, scaleshift_const_shape, {}, true);
std::shared_ptr<ov::Node> scale_const = ngraph::builder::makeConstant<float>(decompression_precision, scaleshift_const_shape, {}, true);
if (reshape_on_decompression_constant) {
auto scale_reshape_const = ov::opset10::Constant::create(ov::element::i32, {scaleshift_target_shape.size()}, scaleshift_target_shape);
auto scale_reshape = std::make_shared<ov::opset10::Reshape>(scale_const, scale_reshape_const, false);
@ -175,6 +188,9 @@ protected:
auto target_shape_node = ov::opset10::Constant::create(ov::element::i32, {reshape_target_shape.size()}, reshape_target_shape);
last_node = std::make_shared<ov::opset10::Reshape>(last_node, target_shape_node, false);
}
if (decompression_precision != data_precision) {
last_node = std::make_shared<ov::opset10::Convert>(last_node, data_precision);
}
if (transpose_weights) {
const size_t rank = last_node->get_output_partial_shape(0).size();
std::vector<int> order(rank);
@ -191,6 +207,7 @@ protected:
const int group_size,
const ov::element::Type data_precision,
const ov::element::Type weights_precision,
const ov::element::Type decompression_precision,
const bool transpose_weights,
const bool add_subtract,
const bool reshape_on_decompression) {
@ -199,6 +216,7 @@ protected:
group_size,
data_precision,
weights_precision,
decompression_precision,
transpose_weights,
add_subtract,
reshape_on_decompression);
@ -211,6 +229,7 @@ protected:
ShapeParams shape_params;
ov::test::ElementType weights_precision;
ov::test::ElementType decompression_precision;
bool transpose_weights;
bool decompression_sub;
bool reshape_on_decompression;
@ -220,6 +239,7 @@ protected:
std::tie(shape_params,
weights_precision,
decompression_precision,
transpose_weights,
decompression_sub,
reshape_on_decompression,
@ -239,6 +259,7 @@ protected:
shape_params.weights_group_size,
netType,
weights_precision,
decompression_precision,
transpose_weights,
decompression_sub,
reshape_on_decompression);
@ -260,7 +281,7 @@ protected:
}
ASSERT_TRUE(weights_found);
const bool should_fuse = std::get<7>(test_param);
const bool should_fuse = std::get<8>(test_param);
const size_t expected_count = should_fuse ? 0 : 1;
CheckNumberOfNodesWithType(compiledModel, "Convert", expected_count);
CheckNumberOfNodesWithType(compiledModel, "Eltwise", expected_count);
@ -304,6 +325,7 @@ bool shouldUseDecompressionKernelBasic() {
}
const std::vector<ov::test::ElementType> weights_precisions = {ov::element::u8, ov::element::nf4};
const std::vector<ov::test::ElementType> decompression_precisions = {ov::element::f32};
const std::vector<ShapeParams> input_shapes_basic = {
{{{-1, -1, -1}, {{1, 4, 16}, {10, 16, 16}}}, {16, 32}},
{{{}, {{1, 4, 16}}}, {16, 32}, 2ul},
@ -331,6 +353,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_basic,
MatmulWeightsDecompression,
::testing::Combine(::testing::ValuesIn(input_shapes_basic),
::testing::ValuesIn(weights_precisions),
::testing::ValuesIn(decompression_precisions),
::testing::Values(true),
::testing::Values(true),
::testing::Values(true),
@ -343,6 +366,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_big,
MatmulWeightsDecompression,
::testing::Combine(::testing::ValuesIn(input_shapes_big),
::testing::ValuesIn(weights_precisions),
::testing::ValuesIn(decompression_precisions),
::testing::Values(true),
::testing::Values(true),
::testing::Values(true),
@ -364,11 +388,13 @@ const std::vector<ShapeParams> input_shapes_corner_cases_big = {
const std::vector<bool> transpose_weights = {true, false};
const std::vector<bool> add_decompression_sub = {true, false};
const std::vector<bool> reshape_on_decompression = {true, false};
const std::vector<ov::test::ElementType> decompression_precisions_corner_cases = {ov::element::f16, ov::element::f32};
INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_corner_cases_basic,
MatmulWeightsDecompression,
::testing::Combine(::testing::ValuesIn(input_shapes_corner_cases_basic),
::testing::ValuesIn(weights_precisions),
::testing::ValuesIn(decompression_precisions_corner_cases),
::testing::ValuesIn(transpose_weights),
::testing::ValuesIn(add_decompression_sub),
::testing::ValuesIn(reshape_on_decompression),
@ -381,6 +407,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_corner_cases_big,
MatmulWeightsDecompression,
::testing::Combine(::testing::ValuesIn(input_shapes_corner_cases_big),
::testing::ValuesIn(weights_precisions),
::testing::ValuesIn(decompression_precisions_corner_cases),
::testing::ValuesIn(transpose_weights),
::testing::ValuesIn(add_decompression_sub),
::testing::ValuesIn(reshape_on_decompression),