From efa2debea2d0b1ea7013d8a070b714f73629bba8 Mon Sep 17 00:00:00 2001 From: Vladimir Paramuzov Date: Tue, 7 May 2024 01:35:26 -0700 Subject: [PATCH] [GPU][CPU][Transformations] Reuse MoveEltwiseUpThroughDataMov transform (#23312) ### Details: - Moved `MoveEltwiseUpThroughDataMov` transformation from CPU plugin to common scope - Relaxed precision restrictions in this transform and keep old one in CPU callback only - Added an ability to customize list of allowed ops for `MoveEltwiseUpThroughDataMov` pass - Enable this pass in GPU pipeline in addition to internal pass. cldnn pass can't be fully replaced for now as for some ops we may create multiple cldnn primitives which happens after `MoveEltwiseUpThroughDataMov` pass is called --- .../move_eltwise_up_data_movement.hpp | 24 ++++ .../move_eltwise_up_data_movement.cpp | 128 ++++++++++++++++++ .../move_eltwise_up_data_movement_test.cpp | 127 ++++++++++------- .../pass/move_eltwise_up_data_movement.cpp | 122 ----------------- .../pass/move_eltwise_up_data_movement.hpp | 19 --- .../transformation_pipeline.cpp | 8 +- .../src/graph/include/pass_manager.h | 1 + .../src/plugin/transformations_pipeline.cpp | 18 +++ 8 files changed, 251 insertions(+), 196 deletions(-) create mode 100644 src/common/transformations/include/transformations/common_optimizations/move_eltwise_up_data_movement.hpp create mode 100644 src/common/transformations/src/transformations/common_optimizations/move_eltwise_up_data_movement.cpp rename src/{plugins/intel_cpu/tests/unit/transformations => common/transformations/tests/common_optimizations}/move_eltwise_up_data_movement_test.cpp (71%) delete mode 100644 src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.cpp delete mode 100644 src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp diff --git a/src/common/transformations/include/transformations/common_optimizations/move_eltwise_up_data_movement.hpp b/src/common/transformations/include/transformations/common_optimizations/move_eltwise_up_data_movement.hpp new file mode 100644 index 00000000000..7f41c1694b3 --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/move_eltwise_up_data_movement.hpp @@ -0,0 +1,24 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/type.hpp" +#include "openvino/pass/graph_rewrite.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { + +class TRANSFORMATIONS_API MoveEltwiseUpThroughDataMov : public ov::pass::MatcherPass { +public: + OPENVINO_RTTI("MoveEltwiseUpThroughDataMov", "0"); + MoveEltwiseUpThroughDataMov(std::vector allowed_data_movement_ops = get_default_allowed_ops()); + +private: + static std::vector get_default_allowed_ops(); +}; + +} // namespace pass +} // namespace ov diff --git a/src/common/transformations/src/transformations/common_optimizations/move_eltwise_up_data_movement.cpp b/src/common/transformations/src/transformations/common_optimizations/move_eltwise_up_data_movement.cpp new file mode 100644 index 00000000000..9218c93bac2 --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/move_eltwise_up_data_movement.cpp @@ -0,0 +1,128 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/common_optimizations/move_eltwise_up_data_movement.hpp" + +#include +#include +#include + +#include "itt.hpp" +#include "openvino/core/rt_info.hpp" +#include "openvino/core/type.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/utils/utils.hpp" + +namespace { +bool is_data_movement_operation(const std::shared_ptr& node, + const std::vector& allowed_data_movement_ops) { + for (auto& allowed_type : allowed_data_movement_ops) { + if (node->get_type_info().is_castable(allowed_type)) + return true; + } + + return false; +} + +bool is_scalar_like(const std::shared_ptr& node) { + auto constant_op = std::dynamic_pointer_cast(node); + return constant_op != nullptr && shape_size(constant_op->get_shape()) == 1; +} +} // namespace + +std::vector ov::pass::MoveEltwiseUpThroughDataMov::get_default_allowed_ops() { + return { + ov::op::v0::Squeeze::get_type_info_static(), + ov::op::v0::Unsqueeze::get_type_info_static(), + ov::op::v1::Reshape::get_type_info_static(), + ov::op::v1::Transpose::get_type_info_static(), + ov::op::v0::ShuffleChannels::get_type_info_static(), + ov::op::v7::Roll::get_type_info_static(), + ov::op::v0::ReverseSequence::get_type_info_static(), + ov::op::v0::DepthToSpace::get_type_info_static(), + ov::op::v1::BatchToSpace::get_type_info_static(), + ov::op::v1::Broadcast::get_type_info_static(), + ov::op::v3::Broadcast::get_type_info_static(), + ov::op::v1::Gather::get_type_info_static(), + ov::op::v7::Gather::get_type_info_static(), + ov::op::v8::Gather::get_type_info_static(), + }; +} + +ov::pass::MoveEltwiseUpThroughDataMov::MoveEltwiseUpThroughDataMov( + std::vector allowed_data_movement_ops) { + MATCHER_SCOPE(MoveEltwiseUpThroughDataMov); + auto eltwise_pattern = ov::pass::pattern::wrap_type(ov::pass::pattern::has_static_rank()); + + ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + + auto eltwise = pattern_map.at(eltwise_pattern).get_node_shared_ptr(); + if (transformation_callback(eltwise)) { + return false; + } + + if (eltwise->get_output_target_inputs(0).size() != 1) { + return false; + } + + for (size_t i = 1; i < eltwise->get_input_size(); ++i) { + if (!is_scalar_like(eltwise->get_input_node_shared_ptr(i))) { + return false; + } + } + + auto current = eltwise->get_input_node_shared_ptr(0); + auto child = eltwise; + + while (is_data_movement_operation(current, allowed_data_movement_ops)) { + if (current->get_output_size() != 1 || current->get_output_target_inputs(0).size() != 1 || + current->get_output_element_type(0) != current->get_input_element_type(0)) { + return false; + } + + child = current; + current = current->get_input_node_shared_ptr(0); + } + + // now current is the first not data movement op + if (child == eltwise) { + return false; + } + + // eltwise constant shape should match new input shape + for (size_t i = 1; i < eltwise->get_input_size(); i++) { + if (current->get_output_partial_shape(0).size() != eltwise->get_input_partial_shape(i).size()) { + auto old_eltwise_const = ov::as_type_ptr(eltwise->get_input_node_shared_ptr(i)); + if (old_eltwise_const->get_shape().size() != 0) { + auto new_constant = std::make_shared(*old_eltwise_const.get(), ov::Shape{}); + ov::replace_node_update_name(old_eltwise_const, new_constant); + } + } + } + ov::replace_output_update_name(eltwise->output(0), eltwise->input_value(0)); + + ov::OutputVector eltwise_inputs = eltwise->input_values(); + eltwise_inputs[0] = child->input_value(0); + auto new_eltwise = eltwise->clone_with_new_inputs(eltwise_inputs); + // WA: it's necessary to set empty friendly name here + // to avoid name duplication in TypeRelaxed cases + new_eltwise->set_friendly_name(""); + ov::copy_runtime_info(eltwise, new_eltwise); + + ov::OutputVector child_inputs = child->input_values(); + child_inputs[0] = new_eltwise; + auto new_child = child->clone_with_new_inputs(child_inputs); + ov::copy_runtime_info(child, new_child); + new_child->set_friendly_name(child->get_friendly_name()); + + ov::replace_node(child, new_child); + return true; + }; + + auto m = std::make_shared(eltwise_pattern, matcher_name); + register_matcher(m, callback); +} diff --git a/src/plugins/intel_cpu/tests/unit/transformations/move_eltwise_up_data_movement_test.cpp b/src/common/transformations/tests/common_optimizations/move_eltwise_up_data_movement_test.cpp similarity index 71% rename from src/plugins/intel_cpu/tests/unit/transformations/move_eltwise_up_data_movement_test.cpp rename to src/common/transformations/tests/common_optimizations/move_eltwise_up_data_movement_test.cpp index 8b586676ce2..b2561d5f1fc 100644 --- a/src/plugins/intel_cpu/tests/unit/transformations/move_eltwise_up_data_movement_test.cpp +++ b/src/common/transformations/tests/common_optimizations/move_eltwise_up_data_movement_test.cpp @@ -2,20 +2,21 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "transformations/common_optimizations/move_eltwise_up_data_movement.hpp" + #include #include #include -#include "ov_ops/type_relaxed.hpp" -#include #include #include "common_test_utils/ov_test_utils.hpp" -#include +#include "ov_ops/type_relaxed.hpp" +#include "transformations/init_node_info.hpp" using namespace testing; -class MoveEltwiseUpThroughDataMovTest: public TransformationTestsF{}; +class MoveEltwiseUpThroughDataMovTest : public TransformationTestsF {}; TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwise) { const ov::Shape shape{1, 3, 224, 224}; @@ -24,7 +25,8 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwise) { { auto input = std::make_shared(ov::element::f32, shape); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(input, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); @@ -33,14 +35,15 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwise) { auto sigmoid = std::make_shared(unsqueeze); model = std::make_shared(ov::NodeVector{sigmoid}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } { auto input = std::make_shared(ov::element::f32, shape); auto sigmoid = std::make_shared(input); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(sigmoid, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); @@ -65,7 +68,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, TypeRelaxedEltwise) { auto multiply = std::make_shared>(transpose, mul_const); model = std::make_shared(ov::NodeVector{multiply}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } { auto input = std::make_shared(ov::element::f32, shape); @@ -74,11 +77,11 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, TypeRelaxedEltwise) { auto mul_const = ov::opset8::Constant::create(ov::element::f32, {}, {2.f}); auto multiply = std::make_shared>(intermediate_op, mul_const); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(multiply, transpose_const); - model_ref = - std::make_shared(ov::NodeVector{transpose}, ov::ParameterVector{input}); + model_ref = std::make_shared(ov::NodeVector{transpose}, ov::ParameterVector{input}); } } @@ -92,7 +95,8 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, EltwiseSequence) { auto matmul = std::make_shared(input_left, input_right); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(matmul, transpose_const); auto relu = std::make_shared(transpose); @@ -103,7 +107,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, EltwiseSequence) { auto sigmoid = std::make_shared(unsqueeze); model = std::make_shared(ov::NodeVector{sigmoid}, ov::ParameterVector{input_left, input_right}); - manager.register_pass(); + manager.register_pass(); } { auto input_left = std::make_shared(ov::element::f32, shape); @@ -115,13 +119,15 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, EltwiseSequence) { auto sigmoid = std::make_shared(relu); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(sigmoid, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(transpose, unsqueeze_const); - model_ref = std::make_shared(ov::NodeVector{unsqueeze}, ov::ParameterVector{input_left, input_right}); + model_ref = + std::make_shared(ov::NodeVector{unsqueeze}, ov::ParameterVector{input_left, input_right}); } } @@ -147,7 +153,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, DataMovementTwoConsumers) { auto relu = std::make_shared(transpose); model = std::make_shared(ov::NodeVector{sigmoid, relu}, ov::ParameterVector{input_left, input_right}); - manager.register_pass(); + manager.register_pass(); } TEST_F(MoveEltwiseUpThroughDataMovTest, SingleBinaryEltwiseWithScalarOnSecondBranch) { @@ -158,23 +164,29 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleBinaryEltwiseWithScalarOnSecondBra { auto input = std::make_shared(ov::element::f32, shape); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(input, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(transpose, unsqueeze_const); - auto add = std::make_shared(unsqueeze, ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); + auto add = + std::make_shared(unsqueeze, + ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); - manager.register_pass(); + manager.register_pass(); model = std::make_shared(ov::NodeVector{add}, ov::ParameterVector{input}); } { auto input = std::make_shared(ov::element::f32, shape); - auto add = std::make_shared(input, ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); + auto add = + std::make_shared(input, + ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(add, transpose_const); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); @@ -195,15 +207,19 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleEltwiseWith5ScalarOnSecondBranch) auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(input, unsqueeze_const); - auto add = std::make_shared(unsqueeze, ov::opset8::Constant::create(ov::element::f32, {1, 1, 1, 1, 1}, {scalar_value})); + auto add = std::make_shared( + unsqueeze, + ov::opset8::Constant::create(ov::element::f32, {1, 1, 1, 1, 1}, {scalar_value})); - manager.register_pass(); + manager.register_pass(); model = std::make_shared(ov::NodeVector{add}, ov::ParameterVector{input}); } { auto input = std::make_shared(ov::element::f32, shape); - auto add = std::make_shared(input, ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); + auto add = + std::make_shared(input, + ov::opset8::Constant::create(ov::element::f32, {}, {scalar_value})); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(add, unsqueeze_const); @@ -229,7 +245,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleBinaryEltwiseWithNotScalarOnSecond auto add = std::make_shared(unsqueeze, add_scalar); model = std::make_shared(ov::NodeVector{add}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwiseDynamicShape) { @@ -244,7 +260,7 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwiseDynamicShape) { auto sigmoid = std::make_shared(unsqueeze); model = std::make_shared(ov::NodeVector{sigmoid}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } { @@ -263,13 +279,14 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwiseDynamicRank) { const std::vector input_order = {3, 2, 1, 0}; const int64_t unsqueeze_axis = 2; - auto input = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(ov::Rank::dynamic())); + auto input = + std::make_shared(ov::element::f32, ov::PartialShape::dynamic(ov::Rank::dynamic())); auto unsqueeze_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}, {unsqueeze_axis}); auto unsqueeze = std::make_shared(input, unsqueeze_const); auto sigmoid = std::make_shared(unsqueeze); model = std::make_shared(ov::NodeVector{sigmoid}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } TEST_F(MoveEltwiseUpThroughDataMovTest, TransposeFakeQuantize) { @@ -278,28 +295,32 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, TransposeFakeQuantize) { { auto input = std::make_shared(ov::element::f32, shape); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(input, transpose_const); - auto fakequantize = std::make_shared(transpose, - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-8.5}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {8.5}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), - 255); + auto fakequantize = std::make_shared( + transpose, + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-8.5}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {8.5}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), + 255); model = std::make_shared(ov::NodeVector{fakequantize}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } { auto input = std::make_shared(ov::element::f32, shape); - auto fakequantize = std::make_shared(input, - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-8.5}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {8.5}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), - 255); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto fakequantize = std::make_shared( + input, + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-8.5}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {8.5}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), + 255); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(fakequantize, transpose_const); model_ref = std::make_shared(ov::NodeVector{transpose}, ov::ParameterVector{input}); @@ -312,17 +333,19 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, TransposeFakeQuantizePerChannel) { { auto input = std::make_shared(ov::element::f32, shape); - auto transpose_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); + auto transpose_const = + ov::opset8::Constant::create(ov::element::i64, ov::Shape{input_order.size()}, input_order); auto transpose = std::make_shared(input, transpose_const); - auto fakequantize = std::make_shared(transpose, - ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 3, 1, 1}, {-8.5, -7.5, -10.}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 3, 1, 1}, {8.5, 7.5, 10.}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), - 255); + auto fakequantize = std::make_shared( + transpose, + ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 3, 1, 1}, {-8.5, -7.5, -10.}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{1, 3, 1, 1}, {8.5, 7.5, 10.}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {-128}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{}, {127}), + 255); model = std::make_shared(ov::NodeVector{fakequantize}, ov::ParameterVector{input}); - manager.register_pass(); + manager.register_pass(); } -} \ No newline at end of file +} diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.cpp deleted file mode 100644 index 98b30052efd..00000000000 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "move_eltwise_up_data_movement.hpp" - -#include -#include -#include - -#include -#include "openvino/core/rt_info.hpp" -#include "openvino/pass/pattern/op/wrap_type.hpp" -#include "transformations/utils/utils.hpp" - -#include "itt.hpp" - - -namespace { - bool is_data_movement_operation(const std::shared_ptr& node) { - return ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node) || - ov::is_type(node); - } - - bool is_scalar_like(const std::shared_ptr& node) { - auto constantNode = std::dynamic_pointer_cast(node); - return constantNode != nullptr && shape_size(constantNode->get_shape()) == 1; - } -} // namespace - -ov::intel_cpu::MoveEltwiseUpThroughDataMov::MoveEltwiseUpThroughDataMov() { - MATCHER_SCOPE(MoveEltwiseUpThroughDataMov); - auto eltwise_pattern = ov::pass::pattern::wrap_type(ov::pass::pattern::has_static_rank()); - - ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { - const auto& pattern_map = m.get_pattern_value_map(); - - auto eltwise = pattern_map.at(eltwise_pattern).get_node_shared_ptr(); - if (transformation_callback(eltwise)) { - return false; - } - - if (eltwise->get_output_target_inputs(0).size() != 1) { - return false; - } - - for (size_t i = 1; i < eltwise->get_input_size(); ++i) { - if (!is_scalar_like(eltwise->get_input_node_shared_ptr(i))) { - return false; - } - } - - if (!ov::is_type(eltwise) && eltwise->get_output_element_type(0) != eltwise->get_input_element_type(0)) { - return false; - } - - auto current = eltwise->get_input_node_shared_ptr(0); - auto child = eltwise; - - while (is_data_movement_operation(current)) { - if (current->get_output_size() != 1 || - current->get_output_target_inputs(0).size() != 1 || - current->get_output_element_type(0) != current->get_input_element_type(0)) { - return false; - } - - child = current; - current = current->get_input_node_shared_ptr(0); - } - - // now current is the first not data movement op - if (child == eltwise) { - return false; - } - - // eltwise constant shape should match new input shape - for (size_t i = 1; i < eltwise->get_input_size(); i++) { - if (current->get_output_partial_shape(0).size() != eltwise->get_input_partial_shape(i).size()) { - auto old_eltwise_const = ov::as_type_ptr(eltwise->get_input_node_shared_ptr(i)); - if (old_eltwise_const->get_shape().size() != 0) { - auto new_constant = std::make_shared(*old_eltwise_const.get(), ov::Shape{}); - ov::replace_node_update_name(old_eltwise_const, new_constant); - } - } - } - ov::replace_output_update_name(eltwise->output(0), eltwise->input_value(0)); - - ov::OutputVector eltwiseInputs = eltwise->input_values(); - eltwiseInputs[0] = child->input_value(0); - auto newEltwise = eltwise->clone_with_new_inputs(eltwiseInputs); - // WA: it's necessary to set empty friendly name here - // to avoid name duplication in TypeRelaxed cases - newEltwise->set_friendly_name(""); - ov::copy_runtime_info(eltwise, newEltwise); - - ov::OutputVector childInputs = child->input_values(); - childInputs[0] = newEltwise; - auto newChild = child->clone_with_new_inputs(childInputs); - ov::copy_runtime_info(child, newChild); - newChild->set_friendly_name(child->get_friendly_name()); - - ov::replace_node(child, newChild); - return true; - }; - - auto m = std::make_shared(eltwise_pattern, matcher_name); - register_matcher(m, callback); -} diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp deleted file mode 100644 index 296fae9b779..00000000000 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "openvino/pass/graph_rewrite.hpp" - -namespace ov { -namespace intel_cpu { - -class MoveEltwiseUpThroughDataMov : public ov::pass::MatcherPass { -public: - OPENVINO_RTTI("MoveEltwiseUpThroughDataMov", "0"); - MoveEltwiseUpThroughDataMov(); -}; - -} // namespace intel_cpu -} // namespace ov diff --git a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index 6a2ee84db0d..00ac58fcfc4 100644 --- a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -33,6 +33,7 @@ #include "transformations/common_optimizations/wrap_interpolate_into_transposes.hpp" #include "transformations/common_optimizations/matmul_const_transposes_extraction.hpp" #include "transformations/common_optimizations/fuse_rotary_positional_embeddings.hpp" +#include "transformations/common_optimizations/move_eltwise_up_data_movement.hpp" #include "transformations/control_flow/unroll_tensor_iterator.hpp" #include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/op_conversions/convert_batch_to_space.hpp" @@ -116,7 +117,6 @@ #include "transformations/cpu_opset/common/pass/decompose_integer_divide.hpp" #include "transformations/cpu_opset/common/pass/convert_fq_rnn_to_quantized_rnn.hpp" #include "transformations/cpu_opset/common/pass/insert_convert_after_extension.hpp" -#include "transformations/cpu_opset/common/pass/move_eltwise_up_data_movement.hpp" #include "transformations/cpu_opset/common/pass/ngram_fusion.hpp" #include "transformations/cpu_opset/common/pass/permute_slice_n_interpolation.hpp" #include "transformations/cpu_opset/common/pass/swap_convert_transpose.hpp" @@ -744,9 +744,11 @@ void Transformations::PostLpt() { return node->get_rt_info().count("UNROLL_TI") == 0; }, ov::pass::UnrollTensorIterator); - CPU_REGISTER_PASS_COMMON(postLPTPassManager, MoveEltwiseUpThroughDataMov); + CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::MoveEltwiseUpThroughDataMov); CPU_SET_CALLBACK_COMMON(postLPTPassManager, [](const std::shared_ptr& node) -> bool { + if (!ov::is_type(node) && node->get_output_element_type(0) != node->get_input_element_type(0)) + return true; if (node->get_input_size() >= 2) { return node->get_input_element_type(1) == ov::element::i8 || node->get_input_element_type(1) == ov::element::u8 || @@ -754,7 +756,7 @@ void Transformations::PostLpt() { } return false; }, - MoveEltwiseUpThroughDataMov); + ov::pass::MoveEltwiseUpThroughDataMov); CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::Validate); CPU_REGISTER_PASS_COMMON(postLPTPassManager, ov::pass::ConstantFolding); diff --git a/src/plugins/intel_gpu/src/graph/include/pass_manager.h b/src/plugins/intel_gpu/src/graph/include/pass_manager.h index 2a94c0cd782..764f91893b8 100644 --- a/src/plugins/intel_gpu/src/graph/include/pass_manager.h +++ b/src/plugins/intel_gpu/src/graph/include/pass_manager.h @@ -163,6 +163,7 @@ private: void conv_eltwise_read_write_opt(program& p, program_node* node); }; +// TODO: Remove this pass once no unexpected reshapes/reorders are added during ov::Model -> cldnn::topology conversion class prepare_primitive_fusing_through : public base_pass { public: prepare_primitive_fusing_through() : base_pass("prepare_primitive_fusing_through") {} diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 94e90b475af..865972abda8 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -41,8 +41,11 @@ #include "openvino/op/reduce_max.hpp" #include "openvino/op/reduce_mean.hpp" #include "openvino/op/reduce_sum.hpp" +#include "openvino/op/reshape.hpp" #include "openvino/op/rnn_cell.hpp" #include "openvino/op/rnn_sequence.hpp" +#include "openvino/op/squeeze.hpp" +#include "openvino/op/unsqueeze.hpp" #include "openvino/op/util/sub_graph_base.hpp" #include "openvino/pass/constant_folding.hpp" #include "openvino/pass/manager.hpp" @@ -67,6 +70,7 @@ #include "transformations/common_optimizations/convert_quantize_dequantize.hpp" #include "transformations/common_optimizations/lin_op_sequence_fusion.hpp" #include "transformations/common_optimizations/lstm_cell_fusion.hpp" +#include "transformations/common_optimizations/move_eltwise_up_data_movement.hpp" #include "transformations/common_optimizations/mvn_fusion.hpp" #include "transformations/common_optimizations/softmax_fusion.hpp" #include "transformations/common_optimizations/transpose_sinking.hpp" @@ -705,6 +709,20 @@ void TransformationsPipeline::apply(std::shared_ptr func) { { ov::pass::Manager manager; + + // Other ops support eltwise fusions + const std::vector allowed_data_movement_ops = { + ov::op::v1::Reshape::get_type_info_static(), + ov::op::v0::Squeeze::get_type_info_static(), + ov::op::v0::Unsqueeze::get_type_info_static(), + ov::op::v0::ShuffleChannels::get_type_info_static(), + ov::op::v7::Roll::get_type_info_static(), + ov::op::v0::ReverseSequence::get_type_info_static(), + ov::op::v1::Broadcast::get_type_info_static(), + ov::op::v3::Broadcast::get_type_info_static(), + }; + manager.register_pass(allowed_data_movement_ops); + manager.register_pass(); manager.register_pass(); manager.register_pass();