diff --git a/src/common/transformations/include/transformations/common_optimizations/move_eltwise_up_data_movement.hpp b/src/common/transformations/include/transformations/common_optimizations/move_eltwise_up_data_movement.hpp new file mode 100644 index 00000000000..7f41c1694b3 --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/move_eltwise_up_data_movement.hpp @@ -0,0 +1,24 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/type.hpp" +#include "openvino/pass/graph_rewrite.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { + +class TRANSFORMATIONS_API MoveEltwiseUpThroughDataMov : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("MoveEltwiseUpThroughDataMov", "0"); + MoveEltwiseUpThroughDataMov(std::vector allowed_data_movement_ops = get_default_allowed_ops()); + +private: + static std::vector get_default_allowed_ops(); +}; + +} // namespace pass +} // namespace ov diff --git a/src/common/transformations/src/transformations/common_optimizations/move_eltwise_up_data_movement.cpp b/src/common/transformations/src/transformations/common_optimizations/move_eltwise_up_data_movement.cpp new file mode 100644 index 00000000000..9218c93bac2 --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/move_eltwise_up_data_movement.cpp @@ -0,0 +1,128 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/common_optimizations/move_eltwise_up_data_movement.hpp" + +#include +#include +#include + +#include "itt.hpp" +#include "openvino/core/rt_info.hpp" +#include "openvino/core/type.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/utils/utils.hpp" + +namespace { +bool is_data_movement_operation(const std::shared_ptr& node, + const std::vector& allowed_data_movement_ops) { + for (auto& allowed_type : allowed_data_movement_ops) { + if (node->get_type_info().is_castable(allowed_type)) + return true; + } + + return false; +} + +bool is_scalar_like(const std::shared_ptr& node) { + auto constant_op = std::dynamic_pointer_cast(node); + return constant_op != nullptr && shape_size(constant_op->get_shape()) == 1; +} +} // namespace + +std::vector ov::pass::MoveEltwiseUpThroughDataMov::get_default_allowed_ops() { + return { + ov::op::v0::Squeeze::get_type_info_static(), + ov::op::v0::Unsqueeze::get_type_info_static(), + ov::op::v1::Reshape::get_type_info_static(), + ov::op::v1::Transpose::get_type_info_static(), + ov::op::v0::ShuffleChannels::get_type_info_static(), + ov::op::v7::Roll::get_type_info_static(), + ov::op::v0::ReverseSequence::get_type_info_static(), + ov::op::v0::DepthToSpace::get_type_info_static(), + ov::op::v1::BatchToSpace::get_type_info_static(), + ov::op::v1::Broadcast::get_type_info_static(), + ov::op::v3::Broadcast::get_type_info_static(), + ov::op::v1::Gather::get_type_info_static(), + ov::op::v7::Gather::get_type_info_static(), + ov::op::v8::Gather::get_type_info_static(), + }; +} + +ov::pass::MoveEltwiseUpThroughDataMov::MoveEltwiseUpThroughDataMov( + std::vector allowed_data_movement_ops) { + MATCHER_SCOPE(MoveEltwiseUpThroughDataMov); + auto eltwise_pattern = ov::pass::pattern::wrap_type(ov::pass::pattern::has_static_rank()); + + ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + + auto eltwise = pattern_map.at(eltwise_pattern).get_node_shared_ptr(); + if (transformation_callback(eltwise)) { + return false; + } + + if (eltwise->get_output_target_inputs(0).size() != 1) { + return false; + } + + for (size_t i = 1; i < eltwise->get_input_size(); ++i) { + if (!is_scalar_like(eltwise->get_input_node_shared_ptr(i))) { + return false; + } + } + + auto current = eltwise->get_input_node_shared_ptr(0); + auto child = eltwise; + + while (is_data_movement_operation(current, allowed_data_movement_ops)) { + if (current->get_output_size() != 1 || current->get_output_target_inputs(0).size() != 1 || + current->get_output_element_type(0) != current->get_input_element_type(0)) { + return false; + } + + child = current; + current = current->get_input_node_shared_ptr(0); + } + + // now current is the first not data movement op + if (child == eltwise) { + return false; + } + + // eltwise constant shape should match new input shape + for (size_t i = 1; i < eltwise->get_input_size(); i++) { + if (current->get_output_partial_shape(0).size() != eltwise->get_input_partial_shape(i).size()) { + auto old_eltwise_const = ov::as_type_ptr(eltwise->get_input_node_shared_ptr(i)); + if (old_eltwise_const->get_shape().size() != 0) { + auto new_constant = std::make_shared(*old_eltwise_const.get(), ov::Shape{}); + ov::replace_node_update_name(old_eltwise_const, new_constant); + } + } + } + ov::replace_output_update_name(eltwise->output(0), eltwise->input_value(0)); + + ov::OutputVector eltwise_inputs = eltwise->input_values(); + eltwise_inputs[0] = child->input_value(0); + auto new_eltwise = eltwise->clone_with_new_inputs(eltwise_inputs); + // WA: it's necessary to set empty friendly name here + // to avoid name duplication in TypeRelaxed cases + new_eltwise->set_friendly_name(""); + ov::copy_runtime_info(eltwise, new_eltwise); + + ov::OutputVector child_inputs = child->input_values(); + child_inputs[0] = new_eltwise; + auto new_child = child->clone_with_new_inputs(child_inputs); + ov::copy_runtime_info(child, new_child); + new_child->set_friendly_name(child->get_friendly_name()); + + ov::replace_node(child, new_child); + return true; + }; + + auto m = std::make_shared(eltwise_pattern, matcher_name); + register_matcher(m, callback); +} diff --git a/src/plugins/intel_cpu/tests/unit/transformations/move_eltwise_up_data_movement_test.cpp b/src/common/transformations/tests/common_optimizations/move_eltwise_up_data_movement_test.cpp similarity index 71% rename from src/plugins/intel_cpu/tests/unit/transformations/move_eltwise_up_data_movement_test.cpp rename to src/common/transformations/tests/common_optimizations/move_eltwise_up_data_movement_test.cpp index 8b586676ce2..b2561d5f1fc 100644 --- a/src/plugins/intel_cpu/tests/unit/transformations/move_eltwise_up_data_movement_test.cpp +++ b/src/common/transformations/tests/common_optimizations/move_eltwise_up_data_movement_test.cpp @@ -2,20 +2,21 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "transformations/common_optimizations/move_eltwise_up_data_movement.hpp" + #include #include #include -#include "ov_ops/type_relaxed.hpp" -#include #include #include "common_test_utils/ov_test_utils.hpp" -#include +#include "ov_ops/type_relaxed.hpp" +#include "transformations/init_node_info.hpp" using namespace testing; -class MoveEltwiseUpThroughDataMovTest: public TransformationTestsF{}; +class MoveEltwiseUpThroughDataMovTest : public TransformationTestsF {}; TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwise) { const ov::Shape shape{1, 3, 224, 224}; @@ -24,7 +25,8 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwise) { { auto input = std::make_shared(ov::element::f32, shape); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(input, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); @@ -33,14 +35,15 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwise) { auto sigmoid = std::make_shared(unsqueeze); model = std::make_shared(ov::NodeVector{sigmoid}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } { auto input = std::make_shared(ov::element::f32, shape); auto sigmoid = std::make_shared(input); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(sigmoid, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); @@ -65,7 +68,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, TypeRelaxedEltwise) { auto multiply = std::make_shared>(transpose, mul_const); model = std::make_shared(ov::NodeVector{multiply}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } { auto input = std::make_shared(ov::element::f32, shape); @@ -74,11 +77,11 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, TypeRelaxedEltwise) { auto mul_const = ov::opset8::Constant::create(ov::element::f32, {}, {2.f}); auto multiply = std::make_shared>(intermediate_op, mul_const); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(multiply, transpose_const); - model_ref = - std::make_shared(ov::NodeVector{transpose}, ov::ParameterVector{input}); + model_ref = std::make_shared(ov::NodeVector{transpose}, ov::ParameterVector{input}); } } @@ -92,7 +95,8 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, EltwiseSequence) { auto matmul = std::make_shared(input_left, input_right); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(matmul, transpose_const); auto relu = std::make_shared(transpose); @@ -103,7 +107,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, EltwiseSequence) { auto sigmoid = std::make_shared(unsqueeze); model = std::make_shared(ov::NodeVector{sigmoid}, ov::ParameterVector{input_left, input_right}); - manager.register_pass(); + manager.register_pass(); } { auto input_left = std::make_shared(ov::element::f32, shape); @@ -115,13 +119,15 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, EltwiseSequence) { auto sigmoid = std::make_shared(relu); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(sigmoid, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(transpose, unsqueeze_const); - model_ref = std::make_shared(ov::NodeVector{unsqueeze}, ov::ParameterVector{input_left, input_right}); + model_ref = + std::make_shared(ov::NodeVector{unsqueeze}, ov::ParameterVector{input_left, input_right}); } } @@ -147,7 +153,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, DataMovementTwoConsumers) { auto relu = std::make_shared(transpose); model = std::make_shared(ov::NodeVector{sigmoid, relu}, ov::ParameterVector{input_left, input_right}); - manager.register_pass(); + manager.register_pass(); } TEST_F(MoveEltwiseUpThroughDataMovTest, SingleBinaryEltwiseWithScalarOnSecondBranch) { @@ -158,23 +164,29 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleBinaryEltwiseWithScalarOnSecondBra { auto input = std::make_shared(ov::element::f32, shape); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(input, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(transpose, unsqueeze_const); - auto add = std::make_shared(unsqueeze, ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); + auto add = + std::make_shared(unsqueeze, + ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); - manager.register_pass(); + manager.register_pass(); model = std::make_shared(ov::NodeVector{add}, ov::ParameterVector{input}); } { auto input = std::make_shared(ov::element::f32, shape); - auto add = std::make_shared(input, ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); + auto add = + std::make_shared(input, + ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(add, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); @@ -195,15 +207,19 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleEltwiseWith5ScalarOnSecondBranch) auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(input, unsqueeze_const); - auto add = std::make_shared(unsqueeze, ov::opset8::Constant::create(ov::element::f32, {1, 1, 1, 1, 1}, {scalar_value})); + auto add = std::make_shared( + unsqueeze, + ov::opset8::Constant::create(ov::element::f32, {1, 1, 1, 1, 1}, {scalar_value})); - manager.register_pass(); + manager.register_pass(); model = std::make_shared(ov::NodeVector{add}, ov::ParameterVector{input}); } { auto input = std::make_shared(ov::element::f32, shape); - auto add = std::make_shared(input, ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); + auto add = + std::make_shared(input, + ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(add, unsqueeze_const); @@ -229,7 +245,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleBinaryEltwiseWithNotScalarOnSecond auto add = std::make_shared(unsqueeze, add_scalar); model = std::make_shared(ov::NodeVector{add}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwiseDynamicShape) { @@ -244,7 +260,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwiseDynamicShape) { auto sigmoid = std::make_shared(unsqueeze); model = std::make_shared(ov::NodeVector{sigmoid}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } { @@ -263,13 +279,14 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwiseDynamicRank) { const std::vector input_order = {3, 2, 1, 0}; const int64_t unsqueeze_axis = 2; - auto input = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(ov::Rank::dynamic())); + auto input = + std::make_shared(ov::element::f32, ov::PartialShape::dynamic(ov::Rank::dynamic())); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(input, unsqueeze_const); auto sigmoid = std::make_shared(unsqueeze); model = std::make_shared(ov::NodeVector{sigmoid}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } TEST_F(MoveEltwiseUpThroughDataMovTest, TransposeFakeQuantize) { @@ -278,28 +295,32 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, TransposeFakeQuantize) { { auto input = std::make_shared(ov::element::f32, shape); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(input, transpose_const); - auto fakequantize = std::make_shared(transpose, - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-8.5}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {8.5}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), - 255); + auto fakequantize = std::make_shared( + transpose, + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-8.5}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {8.5}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), + 255); model = std::make_shared(ov::NodeVector{fakequantize}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } { auto input = std::make_shared(ov::element::f32, shape); - auto fakequantize = std::make_shared(input, - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-8.5}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {8.5}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), - 255); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto fakequantize = std::make_shared( + input, + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-8.5}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {8.5}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), + 255); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(fakequantize, transpose_const); model_ref = std::make_shared(ov::NodeVector{transpose}, ov::ParameterVector{input}); @@ -312,17 +333,19 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, TransposeFakeQuantizePerChannel) { { auto input = std::make_shared(ov::element::f32, shape); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(input, transpose_const); - auto fakequantize = std::make_shared(transpose, - ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 3, 1, 1}, {-8.5, -7.5, -10.}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 3, 1, 1}, {8.5, 7.5, 10.}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), - 255); + auto fakequantize = std::make_shared( + transpose, + ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 3, 1, 1}, {-8.5, -7.5, -10.}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 3, 1, 1}, {8.5, 7.5, 10.}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), + 255); model = std::make_shared(ov::NodeVector{fakequantize}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } -} \ No newline at end of file +} diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.cpp deleted file mode 100644 index 98b30052efd..00000000000 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "move_eltwise_up_data_movement.hpp" - -#include -#include -#include - -#include -#include "openvino/core/rt_info.hpp" -#include "openvino/pass/pattern/op/wrap_type.hpp" -#include "transformations/utils/utils.hpp" - -#include "itt.hpp" - - -namespace { - bool is_data_movement_operation(const std::shared_ptr& node) { - return ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node); - } - - bool is_scalar_like(const std::shared_ptr& node) { - auto constantNode = std::dynamic_pointer_cast(node); - return constantNode != nullptr && shape_size(constantNode->get_shape()) == 1; - } -} // namespace - -ov::intel_cpu::MoveEltwiseUpThroughDataMov::MoveEltwiseUpThroughDataMov() { - MATCHER_SCOPE(MoveEltwiseUpThroughDataMov); - auto eltwise_pattern = ov::pass::pattern::wrap_type(ov::pass::pattern::has_static_rank()); - - ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { - const auto& pattern_map = m.get_pattern_value_map(); - - auto eltwise = pattern_map.at(eltwise_pattern).get_node_shared_ptr(); - if (transformation_callback(eltwise)) { - return false; - } - - if (eltwise->get_output_target_inputs(0).size() != 1) { - return false; - } - - for (size_t i = 1; i < eltwise->get_input_size(); ++i) { - if (!is_scalar_like(eltwise->get_input_node_shared_ptr(i))) { - return false; - } - } - - if (!ov::is_type(eltwise) && eltwise->get_output_element_type(0) != eltwise->get_input_element_type(0)) { - return false; - } - - auto current = eltwise->get_input_node_shared_ptr(0); - auto child = eltwise; - - while (is_data_movement_operation(current)) { - if (current->get_output_size() != 1 || - current->get_output_target_inputs(0).size() != 1 || - current->get_output_element_type(0) != current->get_input_element_type(0)) { - return false; - } - - child = current; - current = current->get_input_node_shared_ptr(0); - } - - // now current is the first not data movement op - if (child == eltwise) { - return false; - } - - // eltwise constant shape should match new input shape - for (size_t i = 1; i < eltwise->get_input_size(); i++) { - if (current->get_output_partial_shape(0).size() != eltwise->get_input_partial_shape(i).size()) { - auto old_eltwise_const = ov::as_type_ptr(eltwise->get_input_node_shared_ptr(i)); - if (old_eltwise_const->get_shape().size() != 0) { - auto new_constant = std::make_shared(*old_eltwise_const.get(), ov::Shape{}); - ov::replace_node_update_name(old_eltwise_const, new_constant); - } - } - } - ov::replace_output_update_name(eltwise->output(0), eltwise->input_value(0)); - - ov::OutputVector eltwiseInputs = eltwise->input_values(); - eltwiseInputs[0] = child->input_value(0); - auto newEltwise = eltwise->clone_with_new_inputs(eltwiseInputs); - // WA: it's necessary to set empty friendly name here - // to avoid name duplication in TypeRelaxed cases - newEltwise->set_friendly_name(""); - ov::copy_runtime_info(eltwise, newEltwise); - - ov::OutputVector childInputs = child->input_values(); - childInputs[0] = newEltwise; - auto newChild = child->clone_with_new_inputs(childInputs); - ov::copy_runtime_info(child, newChild); - newChild->set_friendly_name(child->get_friendly_name()); - - ov::replace_node(child, newChild); - return true; - }; - - auto m = std::make_shared(eltwise_pattern, matcher_name); - register_matcher(m, callback); -} diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp deleted file mode 100644 index 296fae9b779..00000000000 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "openvino/pass/graph_rewrite.hpp" - -namespace ov { -namespace intel_cpu { - -class MoveEltwiseUpThroughDataMov : public ov::pass::MatcherPass { -public: - OPENVINO_RTTI("MoveEltwiseUpThroughDataMov", "0"); - MoveEltwiseUpThroughDataMov(); -}; - -} // namespace intel_cpu -} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index 6a2ee84db0d..00ac58fcfc4 100644 --- a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -33,6 +33,7 @@ #include "transformations/common_optimizations/wrap_interpolate_into_transposes.hpp" #include "transformations/common_optimizations/matmul_const_transposes_extraction.hpp" #include "transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp" +#include "transformations/common_optimizations/move_eltwise_up_data_movement.hpp" #include "transformations/control_flow/unroll_tensor_iterator.hpp" #include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/op_conversions/convert_batch_to_space.hpp" @@ -116,7 +117,6 @@ #include "transformations/cpu_opset/common/pass/decompose_integer_divide.hpp" #include "transformations/cpu_opset/common/pass/convert_fq_rnn_to_quantized_rnn.hpp" #include "transformations/cpu_opset/common/pass/insert_convert_after_extension.hpp" -#include "transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp" #include "transformations/cpu_opset/common/pass/ngram_fusion.hpp" #include "transformations/cpu_opset/common/pass/permute_slice_n_interpolation.hpp" #include "transformations/cpu_opset/common/pass/swap_convert_transpose.hpp" @@ -744,9 +744,11 @@ void Transformations::PostLpt() { return node->get_rt_info().count("UNROLL_TI") == 0; }, ov::pass::UnrollTensorIterator); - CPU_REGISTER_PASS_COMMON(postLPTPassManager, MoveEltwiseUpThroughDataMov); + CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::MoveEltwiseUpThroughDataMov); CPU_SET_CALLBACK_COMMON(postLPTPassManager, [](const std::shared_ptr& node) -> bool { + if (!ov::is_type(node) && node->get_output_element_type(0) != node->get_input_element_type(0)) + return true; if (node->get_input_size() >= 2) { return node->get_input_element_type(1) == ov::element::i8 || node->get_input_element_type(1) == ov::element::u8 || @@ -754,7 +756,7 @@ void Transformations::PostLpt() { } return false; }, - MoveEltwiseUpThroughDataMov); + ov::pass::MoveEltwiseUpThroughDataMov); CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::Validate); CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::ConstantFolding); diff --git a/src/plugins/intel_gpu/src/graph/include/pass_manager.h b/src/plugins/intel_gpu/src/graph/include/pass_manager.h index 2a94c0cd782..764f91893b8 100644 --- a/src/plugins/intel_gpu/src/graph/include/pass_manager.h +++ b/src/plugins/intel_gpu/src/graph/include/pass_manager.h @@ -163,6 +163,7 @@ private: void conv_eltwise_read_write_opt(program& p, program_node* node); }; +// TODO: Remove this pass once no unexpected reshapes/reorders are added during ov::Model -> cldnn::topology conversion class prepare_primitive_fusing_through : public base_pass { public: prepare_primitive_fusing_through() : base_pass("prepare_primitive_fusing_through") {} diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 94e90b475af..865972abda8 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -41,8 +41,11 @@ #include "openvino/op/reduce_max.hpp" #include "openvino/op/reduce_mean.hpp" #include "openvino/op/reduce_sum.hpp" +#include "openvino/op/reshape.hpp" #include "openvino/op/rnn_cell.hpp" #include "openvino/op/rnn_sequence.hpp" +#include "openvino/op/squeeze.hpp" +#include "openvino/op/unsqueeze.hpp" #include "openvino/op/util/sub_graph_base.hpp" #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" @@ -67,6 +70,7 @@ #include "transformations/common_optimizations/convert_quantize_dequantize.hpp" #include "transformations/common_optimizations/lin_op_sequence_fusion.hpp" #include "transformations/common_optimizations/lstm_cell_fusion.hpp" +#include "transformations/common_optimizations/move_eltwise_up_data_movement.hpp" #include "transformations/common_optimizations/mvn_fusion.hpp" #include "transformations/common_optimizations/softmax_fusion.hpp" #include "transformations/common_optimizations/transpose_sinking.hpp" @@ -705,6 +709,20 @@ void TransformationsPipeline::apply(std::shared_ptr func) { { ov::pass::Manager manager; + + // Other ops support eltwise fusions + const std::vector allowed_data_movement_ops = { + ov::op::v1::Reshape::get_type_info_static(), + ov::op::v0::Squeeze::get_type_info_static(), + ov::op::v0::Unsqueeze::get_type_info_static(), + ov::op::v0::ShuffleChannels::get_type_info_static(), + ov::op::v7::Roll::get_type_info_static(), + ov::op::v0::ReverseSequence::get_type_info_static(), + ov::op::v1::Broadcast::get_type_info_static(), + ov::op::v3::Broadcast::get_type_info_static(), + }; + manager.register_pass(allowed_data_movement_ops); + manager.register_pass(); manager.register_pass(); manager.register_pass();