natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Keep compressed constants produced by FW (#22095) 

* keep FW 16 bit float constants

* add layer tests

* remove leftovers from onnx, pdpd

* rename to MarkCompressedFloatConstants

* remove VisualizeTree

* added explanation why we need MarkCompressedFloatConstants
This commit is contained in:
Pavel Esir 2024-01-15 12:05:24 +01:00 committed by GitHub
parent 938600fbf7
commit b5b53e1749
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 181 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp
View File

@ -15,6 +15,7 @@ class TRANSFORMATIONS_API EnableDecompressionConvertConstantFolding;
class TRANSFORMATIONS_API DisableDecompressionConvertConstantFolding;
class TRANSFORMATIONS_API KeepConstAndDecompression;
class TRANSFORMATIONS_API KeepConstantsPrecisionAndAddConverts;
class TRANSFORMATIONS_API MarkCompressedFloatConstants;
} // namespace pass
} // namespace ov
@ -58,3 +59,16 @@ public:
OPENVINO_RTTI("KeepConstantsPrecisionAndAddConverts", "0");
KeepConstantsPrecisionAndAddConverts();
};
/**
* @ingroup ie_transformation_common_api
* @brief Prevents ConstantFolding for f16/bf16 Const + Convert_To_FP32 to keep original FW float Constants.
* Original precision should be kept as long as possible, this prevents redundant conversions and saves memory.
* E.g. if original FW model was already compressed no need to upcast during CF, store intermediate f32 consts and
* then again compress them to f16 during save_model.
*/
class ov::pass::MarkCompressedFloatConstants : public MatcherPass {
public:
OPENVINO_RTTI("KeepFWPrecisionFor16BitFloatConstants", "0");
MarkCompressedFloatConstants();
};

27
src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp
View File

@ -119,3 +119,30 @@ pass::KeepConstantsPrecisionAndAddConverts::KeepConstantsPrecisionAndAddConverts
auto m = std::make_shared<pass::pattern::Matcher>(const_pattern, matcher_name);
this->register_matcher(m, callback);
}
pass::MarkCompressedFloatConstants::MarkCompressedFloatConstants() {
MATCHER_SCOPE(MarkCompressedFloatConstants);
auto constant = pattern::wrap_type<ov::op::v0::Constant>();
auto convert = pattern::wrap_type<ov::op::v0::Convert>({constant});
matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& convert_node = as_type_ptr<ov::op::v0::Convert>(m.get_match_root());
const auto& const_node = convert_node->input_value(0).get_node_shared_ptr();
if (convert_node == nullptr || const_node == nullptr)
return false;
if (convert_node->get_destination_type() != element::f32)
return false;
if (const_node->get_output_element_type(0) != element::f16 &&
const_node->get_output_element_type(0) != element::bf16)
return false;
mark_as_decompression(convert_node);
disable_constant_folding(const_node);
disable_constant_folding(convert_node);
return true;
};
auto m = std::make_shared<pass::pattern::Matcher>(convert, matcher_name);
this->register_matcher(m, callback);
}

81
src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp
View File

@ -14,6 +14,7 @@
#include "openvino/opsets/opset8.hpp"
#include "openvino/pass/manager.hpp"
#include "transformations/common_optimizations/mark_precision_sensitive_shapeof_subgraphs.hpp"
#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp"
#include "transformations/init_node_info.hpp"
#include "transformations/utils/utils.hpp"
using namespace ov;
@ -515,3 +516,83 @@ TEST_F(TransformationTestsF, CompressConstants_compress_to_f16_denormal_vals) {
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}
TEST_F(TransformationTestsF, KeepFWPrecisionForFP16Constants_test_1) {
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights = ov::op::v0::Constant::create(
ov::element::f16,
ov::Shape{1, 3, 3, 3},
{1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9});
auto convert_node = std::make_shared<ov::op::v0::Convert>(const_weights, element::f32);
auto conv = std::make_shared<ov::opset8::Convolution>(input,
convert_node,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
model = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
manager.register_pass<ov::pass::MarkCompressedFloatConstants>();
manager.register_pass<ov::pass::CompressFloatConstants>();
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights = ov::opset8::Constant::create(
ov::element::f16,
ov::Shape{1, 3, 3, 3},
{1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9});
auto convert_node = std::make_shared<ov::op::v0::Convert>(const_weights, element::f32);
auto conv = std::make_shared<ov::opset8::Convolution>(input,
convert_node,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
model_ref = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}
TEST_F(TransformationTestsF, KeepFWPrecisionForBF16Constants_test_1) {
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights = ov::op::v0::Constant::create(
ov::element::bf16,
ov::Shape{1, 3, 3, 3},
{1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9});
auto convert_node = std::make_shared<ov::op::v0::Convert>(const_weights, element::f32);
auto conv = std::make_shared<ov::opset8::Convolution>(input,
convert_node,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
model = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
manager.register_pass<ov::pass::MarkCompressedFloatConstants>();
manager.register_pass<ov::pass::CompressFloatConstants>();
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights = ov::opset8::Constant::create(
ov::element::bf16,
ov::Shape{1, 3, 3, 3},
{1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9});
auto convert_node = std::make_shared<ov::op::v0::Convert>(const_weights, element::f32);
auto conv = std::make_shared<ov::opset8::Convolution>(input,
convert_node,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
model_ref = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}

5
src/frontends/pytorch/src/frontend.cpp
View File

@ -16,6 +16,7 @@
#include "transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.hpp"
#include "transformations/common_optimizations/reverse_shape_and_type_infer.hpp"
#include "transformations/control_flow/unroll_if.hpp"
#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp"
#include "transformations/low_precision/mark_dequantization_subgraph.hpp"
#include "transformations/op_conversions/convert_convertlike.hpp"
#include "transformations/resolve_names_collisions.hpp"
@ -176,9 +177,13 @@ void FrontEnd::normalize(const std::shared_ptr<ov::Model>& model) const {
manager.register_pass<ov::pass::ConvertConvertLike>();
manager.register_pass<ov::frontend::pytorch::pass::AtenIndexToSelect>();
// Mark quantized and f16/bf16 compressed constants to prevent CF for them,
// so that not extra memory is used for intermediate decompressed constants.
manager.register_pass<ov::pass::MarkDequantizationSubgraph>(
element::TypeVector{element::u8, element::i8, element::u4, element::i4});
manager.register_pass<ov::pass::MarkCompressedFloatConstants>();
manager.register_pass<ov::pass::ConstantFolding>();
manager.register_pass<ov::frontend::pytorch::pass::AlignTypesRemoval>();
manager.register_pass<ov::pass::PushConstantToSubgraph>();
manager.register_pass<ov::pass::UnrollIf>();

5
src/frontends/tensorflow/src/frontend.cpp
View File

@ -30,6 +30,7 @@
#include "transformations/common_optimizations/remove_concat_zero_dim_input.hpp"
#include "transformations/common_optimizations/reverse_shape_and_type_infer.hpp"
#include "transformations/control_flow/unroll_if.hpp"
#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp"
#include "transformations/resolve_names_collisions.hpp"
#include "transformations/switch_merge_resolve.hpp"
#include "transformations/transpose_sinking/ts_general.hpp"
@ -507,6 +508,10 @@ void FrontEnd::convert(const std::shared_ptr<ov::Model>& partiallyConverted) con
void FrontEnd::normalize(const std::shared_ptr<ov::Model>& model) const {
ov::pass::Manager manager;
// Mark quantized and f16/bf16 compressed constants to prevent CF for them,
// so that not extra memory is used for intermediate decompressed constants.
manager.register_pass<ov::pass::MarkCompressedFloatConstants>();
manager.register_pass<pass::SavedModelUnusedRemover>();
manager.register_pass<pass::EmbeddingSegmentSingleFeatureFusion>();
manager.register_pass<pass::BlockLSTMReplacer>();

4
src/frontends/tensorflow_lite/src/frontend.cpp
View File

@ -16,6 +16,7 @@
#include "tflite_transformations/rfft2d_complex_abs.h"
#include "tflite_transformations/tflite_quantize_resolver.hpp"
#include "transformations/common_optimizations/transpose_sinking.hpp"
#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp"
#include "transformations/resolve_names_collisions.hpp"
#include "transformations/transpose_sinking/ts_general.hpp"
@ -284,6 +285,9 @@ std::shared_ptr<ov::Model> FrontEnd::decode(const InputModel::Ptr& model) const
void FrontEnd::normalize(const std::shared_ptr<ov::Model>& function) const {
ov::pass::Manager manager;
// Mark quantized and f16/bf16 compressed constants to prevent CF for them,
// so that not extra memory is used for intermediate decompressed constants.
manager.register_pass<ov::pass::MarkCompressedFloatConstants>();
manager.register_pass<ov::frontend::tensorflow_lite::pass::TFLQuantizeResolver>();
manager.register_pass<ov::frontend::tensorflow_lite::pass::Rfft2dSimplifier>();
manager.register_pass<ov::pass::TransposeSinking>();

22
tests/layer_tests/ovc_python_api_tests/test_pytorch.py
View File

@ -467,6 +467,27 @@ def create_pytorch_nn_module_scale_list_compression_enabled(tmp_dir):
'compress_to_fp16': True, 'use_convert_model_from_mo': True}
def create_pytorch_nn_module_with_compressed_constants(tmp_dir):
import torch
class NeuralNetwork(torch.nn.Module):
def __init__(self):
super(NeuralNetwork, self).__init__()
self.y = torch.arange(10, dtype=torch.float16)
def forward(self, x):
return x + self.y.to(torch.float32)
param_1 = ov.opset13.parameter([10], dtype=np.float32)
const_1 = ov.opset13.constant(np.arange(10), dtype=np.float16)
convert_1 = ov.opset13.convert(const_1, np.float32)
add_1 = ov.opset13.add(param_1, convert_1)
ov_model_ref = Model([add_1], [param_1], "test")
fw_model = NeuralNetwork()
return fw_model, ov_model_ref, {'input': [([10], np.float32)]}
def create_pytorch_nn_module_shapes_list_static(tmp_dir):
pt_model = make_pt_model_two_inputs()
ref_model = make_ref_pt_model_two_inputs([1, 3, 20, 20])
@ -1020,6 +1041,7 @@ class TestMoConvertPyTorch(CommonMOConvertTest):
create_pytorch_nn_module_scale_list_compression_default,
create_pytorch_nn_module_scale_list_compression_disabled,
create_pytorch_nn_module_scale_list_compression_enabled,
create_pytorch_nn_module_with_compressed_constants,
create_pytorch_nn_module_shapes_list_static,
create_pytorch_nn_module_shapes_list_static_via_input,
create_pytorch_nn_module_shapes_list_dynamic,

23
tests/layer_tests/ovc_python_api_tests/test_tf.py
View File

@ -535,6 +535,28 @@ def create_keras_layer_with_tf_function_call_default_compressed_to_fp16(tmp_dir)
return model, model_ref, {}
def create_keras_layer_with_compressed_constants(tmp_dir):
import tensorflow as tf
class LayerModel(tf.Module):
def __init__(self):
super(LayerModel, self).__init__()
self.const = tf.constant([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], shape=[10], dtype=tf.float16)
@tf.function(input_signature=[tf.TensorSpec([10], tf.float32)])
def __call__(self, input_1):
return input_1 + tf.cast(self.const, dtype=tf.float32)
param_1 = ov.opset13.parameter([10], dtype=np.float32)
const_1 = ov.opset13.constant(np.arange(10), dtype=np.float16)
convert_1 = ov.opset13.convert(const_1, np.float32)
add_1 = ov.opset13.add(param_1, convert_1)
ov_model_ref = Model([add_1], [param_1], "test")
fw_model = LayerModel()
return fw_model, ov_model_ref, {}
def create_keras_layer_with_tf_function_call_no_signature(tmp_dir):
class LayerModel(tf.Module):
def __init__(self):
@ -673,6 +695,7 @@ class TestMoConvertTF(CommonMOConvertTest):
create_keras_layer_with_input_shapes_case4,
create_keras_layer_with_tf_function_call,
create_keras_layer_with_tf_function_call_default_compressed_to_fp16,
create_keras_layer_with_compressed_constants,
create_keras_layer_with_tf_function_call_no_signature,
create_keras_layer_with_tf_function_call_no_signature_single_input,
create_keras_layer_with_string_tensor,