diff --git a/src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp b/src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp index 7576abfac15..037efc27c2f 100644 --- a/src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp +++ b/src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp @@ -15,6 +15,7 @@ class TRANSFORMATIONS_API EnableDecompressionConvertConstantFolding; class TRANSFORMATIONS_API DisableDecompressionConvertConstantFolding; class TRANSFORMATIONS_API KeepConstAndDecompression; class TRANSFORMATIONS_API KeepConstantsPrecisionAndAddConverts; +class TRANSFORMATIONS_API MarkCompressedFloatConstants; } // namespace pass } // namespace ov @@ -58,3 +59,16 @@ public: OPENVINO_RTTI("KeepConstantsPrecisionAndAddConverts", "0"); KeepConstantsPrecisionAndAddConverts(); }; + +/** + * @ingroup ie_transformation_common_api + * @brief Prevents ConstantFolding for f16/bf16 Const + Convert_To_FP32 to keep original FW float Constants. + * Original precision should be kept as long as possible, this prevents redundant conversions and saves memory. + * E.g. if original FW model was already compressed no need to upcast during CF, store intermediate f32 consts and + * then again compress them to f16 during save_model. + */ +class ov::pass::MarkCompressedFloatConstants : public MatcherPass { +public: + OPENVINO_RTTI("KeepFWPrecisionFor16BitFloatConstants", "0"); + MarkCompressedFloatConstants(); +}; diff --git a/src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp b/src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp index 26505dee527..894e8224d5e 100644 --- a/src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp +++ b/src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp @@ -119,3 +119,30 @@ pass::KeepConstantsPrecisionAndAddConverts::KeepConstantsPrecisionAndAddConverts auto m = std::make_shared(const_pattern, matcher_name); this->register_matcher(m, callback); } + +pass::MarkCompressedFloatConstants::MarkCompressedFloatConstants() { + MATCHER_SCOPE(MarkCompressedFloatConstants); + + auto constant = pattern::wrap_type(); + auto convert = pattern::wrap_type({constant}); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& convert_node = as_type_ptr(m.get_match_root()); + const auto& const_node = convert_node->input_value(0).get_node_shared_ptr(); + if (convert_node == nullptr || const_node == nullptr) + return false; + if (convert_node->get_destination_type() != element::f32) + return false; + if (const_node->get_output_element_type(0) != element::f16 && + const_node->get_output_element_type(0) != element::bf16) + return false; + + mark_as_decompression(convert_node); + disable_constant_folding(const_node); + disable_constant_folding(convert_node); + return true; + }; + + auto m = std::make_shared(convert, matcher_name); + this->register_matcher(m, callback); +} diff --git a/src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp b/src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp index 2d256b5df6e..9ed38a8c845 100644 --- a/src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp +++ b/src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp @@ -14,6 +14,7 @@ #include "openvino/opsets/opset8.hpp" #include "openvino/pass/manager.hpp" #include "transformations/common_optimizations/mark_precision_sensitive_shapeof_subgraphs.hpp" +#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/init_node_info.hpp" #include "transformations/utils/utils.hpp" using namespace ov; @@ -515,3 +516,83 @@ TEST_F(TransformationTestsF, CompressConstants_compress_to_f16_denormal_vals) { } comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); } + +TEST_F(TransformationTestsF, KeepFWPrecisionForFP16Constants_test_1) { + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + auto const_weights = ov::op::v0::Constant::create( + ov::element::f16, + ov::Shape{1, 3, 3, 3}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + auto convert_node = std::make_shared(const_weights, element::f32); + + auto conv = std::make_shared(input, + convert_node, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + model = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + + manager.register_pass(); + manager.register_pass(); + } + + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + auto const_weights = ov::opset8::Constant::create( + ov::element::f16, + ov::Shape{1, 3, 3, 3}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + auto convert_node = std::make_shared(const_weights, element::f32); + auto conv = std::make_shared(input, + convert_node, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + model_ref = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + } + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); +} + +TEST_F(TransformationTestsF, KeepFWPrecisionForBF16Constants_test_1) { + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + auto const_weights = ov::op::v0::Constant::create( + ov::element::bf16, + ov::Shape{1, 3, 3, 3}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + auto convert_node = std::make_shared(const_weights, element::f32); + + auto conv = std::make_shared(input, + convert_node, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + model = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + + manager.register_pass(); + manager.register_pass(); + } + + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + auto const_weights = ov::opset8::Constant::create( + ov::element::bf16, + ov::Shape{1, 3, 3, 3}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + auto convert_node = std::make_shared(const_weights, element::f32); + auto conv = std::make_shared(input, + convert_node, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + model_ref = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + } + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); +} diff --git a/src/frontends/pytorch/src/frontend.cpp b/src/frontends/pytorch/src/frontend.cpp index b1e61fbcae9..ee9a9919045 100644 --- a/src/frontends/pytorch/src/frontend.cpp +++ b/src/frontends/pytorch/src/frontend.cpp @@ -16,6 +16,7 @@ #include "transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.hpp" #include "transformations/common_optimizations/reverse_shape_and_type_infer.hpp" #include "transformations/control_flow/unroll_if.hpp" +#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/low_precision/mark_dequantization_subgraph.hpp" #include "transformations/op_conversions/convert_convertlike.hpp" #include "transformations/resolve_names_collisions.hpp" @@ -176,9 +177,13 @@ void FrontEnd::normalize(const std::shared_ptr& model) const { manager.register_pass(); manager.register_pass(); + // Mark quantized and f16/bf16 compressed constants to prevent CF for them, + // so that not extra memory is used for intermediate decompressed constants. manager.register_pass( element::TypeVector{element::u8, element::i8, element::u4, element::i4}); + manager.register_pass(); manager.register_pass(); + manager.register_pass(); manager.register_pass(); manager.register_pass(); diff --git a/src/frontends/tensorflow/src/frontend.cpp b/src/frontends/tensorflow/src/frontend.cpp index c9eafa46890..b017f7d954d 100644 --- a/src/frontends/tensorflow/src/frontend.cpp +++ b/src/frontends/tensorflow/src/frontend.cpp @@ -30,6 +30,7 @@ #include "transformations/common_optimizations/remove_concat_zero_dim_input.hpp" #include "transformations/common_optimizations/reverse_shape_and_type_infer.hpp" #include "transformations/control_flow/unroll_if.hpp" +#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/resolve_names_collisions.hpp" #include "transformations/switch_merge_resolve.hpp" #include "transformations/transpose_sinking/ts_general.hpp" @@ -507,6 +508,10 @@ void FrontEnd::convert(const std::shared_ptr& partiallyConverted) con void FrontEnd::normalize(const std::shared_ptr& model) const { ov::pass::Manager manager; + + // Mark quantized and f16/bf16 compressed constants to prevent CF for them, + // so that not extra memory is used for intermediate decompressed constants. + manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.register_pass(); diff --git a/src/frontends/tensorflow_lite/src/frontend.cpp b/src/frontends/tensorflow_lite/src/frontend.cpp index bb6f82fa552..5f589777436 100644 --- a/src/frontends/tensorflow_lite/src/frontend.cpp +++ b/src/frontends/tensorflow_lite/src/frontend.cpp @@ -16,6 +16,7 @@ #include "tflite_transformations/rfft2d_complex_abs.h" #include "tflite_transformations/tflite_quantize_resolver.hpp" #include "transformations/common_optimizations/transpose_sinking.hpp" +#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/resolve_names_collisions.hpp" #include "transformations/transpose_sinking/ts_general.hpp" @@ -284,6 +285,9 @@ std::shared_ptr FrontEnd::decode(const InputModel::Ptr& model) const void FrontEnd::normalize(const std::shared_ptr& function) const { ov::pass::Manager manager; + // Mark quantized and f16/bf16 compressed constants to prevent CF for them, + // so that not extra memory is used for intermediate decompressed constants. + manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.register_pass(); diff --git a/tests/layer_tests/ovc_python_api_tests/test_pytorch.py b/tests/layer_tests/ovc_python_api_tests/test_pytorch.py index 77a088fea8f..1b5e1eda44c 100644 --- a/tests/layer_tests/ovc_python_api_tests/test_pytorch.py +++ b/tests/layer_tests/ovc_python_api_tests/test_pytorch.py @@ -467,6 +467,27 @@ def create_pytorch_nn_module_scale_list_compression_enabled(tmp_dir): 'compress_to_fp16': True, 'use_convert_model_from_mo': True} +def create_pytorch_nn_module_with_compressed_constants(tmp_dir): + import torch + + class NeuralNetwork(torch.nn.Module): + def __init__(self): + super(NeuralNetwork, self).__init__() + self.y = torch.arange(10, dtype=torch.float16) + + def forward(self, x): + return x + self.y.to(torch.float32) + + param_1 = ov.opset13.parameter([10], dtype=np.float32) + const_1 = ov.opset13.constant(np.arange(10), dtype=np.float16) + convert_1 = ov.opset13.convert(const_1, np.float32) + add_1 = ov.opset13.add(param_1, convert_1) + + ov_model_ref = Model([add_1], [param_1], "test") + fw_model = NeuralNetwork() + return fw_model, ov_model_ref, {'input': [([10], np.float32)]} + + def create_pytorch_nn_module_shapes_list_static(tmp_dir): pt_model = make_pt_model_two_inputs() ref_model = make_ref_pt_model_two_inputs([1, 3, 20, 20]) @@ -1020,6 +1041,7 @@ class TestMoConvertPyTorch(CommonMOConvertTest): create_pytorch_nn_module_scale_list_compression_default, create_pytorch_nn_module_scale_list_compression_disabled, create_pytorch_nn_module_scale_list_compression_enabled, + create_pytorch_nn_module_with_compressed_constants, create_pytorch_nn_module_shapes_list_static, create_pytorch_nn_module_shapes_list_static_via_input, create_pytorch_nn_module_shapes_list_dynamic, diff --git a/tests/layer_tests/ovc_python_api_tests/test_tf.py b/tests/layer_tests/ovc_python_api_tests/test_tf.py index 5f0b8fa5a37..179de425284 100644 --- a/tests/layer_tests/ovc_python_api_tests/test_tf.py +++ b/tests/layer_tests/ovc_python_api_tests/test_tf.py @@ -535,6 +535,28 @@ def create_keras_layer_with_tf_function_call_default_compressed_to_fp16(tmp_dir) return model, model_ref, {} +def create_keras_layer_with_compressed_constants(tmp_dir): + import tensorflow as tf + + class LayerModel(tf.Module): + def __init__(self): + super(LayerModel, self).__init__() + self.const = tf.constant([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], shape=[10], dtype=tf.float16) + + @tf.function(input_signature=[tf.TensorSpec([10], tf.float32)]) + def __call__(self, input_1): + return input_1 + tf.cast(self.const, dtype=tf.float32) + + param_1 = ov.opset13.parameter([10], dtype=np.float32) + const_1 = ov.opset13.constant(np.arange(10), dtype=np.float16) + convert_1 = ov.opset13.convert(const_1, np.float32) + add_1 = ov.opset13.add(param_1, convert_1) + + ov_model_ref = Model([add_1], [param_1], "test") + fw_model = LayerModel() + return fw_model, ov_model_ref, {} + + def create_keras_layer_with_tf_function_call_no_signature(tmp_dir): class LayerModel(tf.Module): def __init__(self): @@ -673,6 +695,7 @@ class TestMoConvertTF(CommonMOConvertTest): create_keras_layer_with_input_shapes_case4, create_keras_layer_with_tf_function_call, create_keras_layer_with_tf_function_call_default_compressed_to_fp16, + create_keras_layer_with_compressed_constants, create_keras_layer_with_tf_function_call_no_signature, create_keras_layer_with_tf_function_call_no_signature_single_input, create_keras_layer_with_string_tensor,