diff --git a/src/plugins/intel_gpu/include/intel_gpu/primitives/eltwise.hpp b/src/plugins/intel_gpu/include/intel_gpu/primitives/eltwise.hpp index 22faf89c3de..982c184646d 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/primitives/eltwise.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/primitives/eltwise.hpp @@ -159,15 +159,6 @@ struct eltwise : public primitive_base { } } - bool needs_onednn_sum_post_op(layout input_layout) const { - if (mode == eltwise_mode::sum && - (input_layout.size.spatial[0] > 1 || input_layout.size.spatial[1] > 1 || input_layout.size.batch[0] > 1)) { - return true; - } - - return false; - } - /// @param mode Eltwise mode. eltwise_mode mode; /// @param coefficients Blob-wise coefficient for SUM operation. diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp index 91fd3387f2d..a9154dbc194 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp @@ -98,7 +98,7 @@ bool concat_in_place_optimization::match(concatenation_node& node) { auto eltw_in_layout = eltw_in.get_output_layout(); auto out_layout = input->get_output_layout(); - if (!fused_op.node->as().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout)) + if (!program_helpers::needs_onednn_sum_post_op(fused_op.node->as(), eltw_in_layout)) continue; if (program_helpers::are_layouts_identical_for_onednn_sum_post_op(eltw_in_layout, out_layout)) return false; diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp index 392ec39a2cf..779eab64e39 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp @@ -636,6 +636,49 @@ void reorder_inputs::run(program& p, layout_optimizer& lo, reorder_factory& rf) conv_node.get_dependencies().front()->set_output_layout(new_layout, false); } } + + // When the conv node is of onednn impl type and eltwise sum with full tensor is fused, + // changes the input format of eltwise sum post-op to use binary add. + if (conv_node.get_preferred_impl_type() == impl_types::onednn) { + std::vector eltw_sum_dep_indices; + for (size_t i = 1; i < conv_node.get_dependencies().size(); i++) { + auto& dep = conv_node.get_dependency(i); + for (auto& fused_op : conv_node.get_fused_primitives()) { + if (fused_op.node->is_type() + && fused_op.node->as().get_primitive()->mode == eltwise_mode::sum + && !program_helpers::needs_onednn_sum_post_op(fused_op.node->as(), + conv_node.get_dependency(fused_op.dep_start_idx).get_output_layout()) + && conv_node.get_dependency(fused_op.dep_start_idx).get_users().size() == 1 + && conv_node.get_dependency(fused_op.dep_start_idx).id() == dep.id()) { + eltw_sum_dep_indices.push_back(i); + } + } + } + + auto conv_layout = conv_node.get_output_layout(); + for (auto& dep_id : eltw_sum_dep_indices) { + auto& prev_node = conv_node.get_dependency(dep_id); + auto old_layout = prev_node.get_output_layout(); + auto expected_format = format::any; + if ((conv_layout.data_type == data_types::f16 || conv_layout.data_type == data_types::f32) + && data_type_traits::is_i8_u8(old_layout.data_type)) { + if (conv_layout.format == format::b_fs_yx_fsv16) + expected_format = format::b_fs_yx_fsv32; + if (conv_layout.format == format::bs_fs_yx_bsv32_fsv16) + expected_format = format::bs_fs_yx_bsv32_fsv32; + } + + if (expected_format != format::any && old_layout.format != expected_format) { + auto new_layout = old_layout; + new_layout.format = expected_format; + auto new_input = rf.get_reorder(prev_node.id(), old_layout, new_layout); + if (new_input.first) { + p.add_intermediate(new_input.first, conv_node, dep_id, !new_input.second); + } + conv_node.get_dependency(dep_id).set_output_layout(new_layout, false); + } + } + } }; const auto reorder_input_fully_connected = [&p, &lo, &rf](typed_program_node& fc_node) { diff --git a/src/plugins/intel_gpu/src/graph/include/program_helpers.h b/src/plugins/intel_gpu/src/graph/include/program_helpers.h index 723486129a8..7b20e82b676 100644 --- a/src/plugins/intel_gpu/src/graph/include/program_helpers.h +++ b/src/plugins/intel_gpu/src/graph/include/program_helpers.h @@ -10,6 +10,7 @@ #include "intel_gpu/runtime/engine.hpp" #include "intel_gpu/graph/program.hpp" #include "data_inst.h" +#include "eltwise_inst.h" #include #include @@ -126,6 +127,8 @@ struct program_helpers { static layout get_weights_layout(typed_program_node& data_node, int32_t split); static bool are_layouts_identical_for_onednn_sum_post_op(layout input_layout, layout output_layout); + + static bool needs_onednn_sum_post_op(const eltwise_node& n, layout input_layout); }; // Base class for performing pattern match style optimizations. diff --git a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp index 56ccb756d2c..3113d80bdff 100644 --- a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp +++ b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp @@ -201,7 +201,7 @@ bool layout_optimizer::can_fuse_reorder(program_node& prev, program_node& next, if (fused_op.node->is_type() && fused_op.deps.size() == 1) { auto eltw_in_layout = next.get_dependency(fused_op.dep_start_idx).get_output_layout(); auto out_layout = next.get_output_layout(); - if (fused_op.node->as().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout) && + if (program_helpers::needs_onednn_sum_post_op(fused_op.node->as(), eltw_in_layout) && program_helpers::are_layouts_identical_for_onednn_sum_post_op(eltw_in_layout, out_layout) && prev.get_output_layout().format != out_layout.format) return false; @@ -959,7 +959,7 @@ layout layout_optimizer::get_expected_layout(layout const& current_layout, auto out_dt = out_layout.data_type; if ((out_layout.count() == in_layout.count()) && (data_type_traits::is_floating_point(in_dt) || data_type_traits::is_floating_point(out_dt)) && in_dt != out_dt && - fo.node->as().get_primitive()->needs_onednn_sum_post_op(in_layout)) { + program_helpers::needs_onednn_sum_post_op(fo.node->as(), in_layout)) { onednn_valid_post_ops = false; break; } @@ -1394,7 +1394,7 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format if (fused_op.node->is_type() && node.get_dependencies().size() > fused_op.dep_start_idx && fused_op.deps.size() == 1) { auto& eltw_in = node.get_dependency(fused_op.dep_start_idx); if (program_helpers::are_layouts_identical_for_onednn_sum_post_op(eltw_in.get_output_layout(), node.get_output_layout()) && - fused_op.node->as().get_primitive()->needs_onednn_sum_post_op(eltw_in.get_output_layout())) { + program_helpers::needs_onednn_sum_post_op(fused_op.node->as(), eltw_in.get_output_layout())) { if (sum_post_op_cnt > 0) return impl_types::ocl; @@ -1448,7 +1448,7 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format auto out_layout = node.get_output_layout(); auto in_dt = in_layout.data_type; auto out_dt = out_layout.data_type; - if (fo.node->as().get_primitive()->needs_onednn_sum_post_op(in_layout)) { + if (program_helpers::needs_onednn_sum_post_op(fo.node->as(), in_layout)) { if ((out_layout.count() == in_layout.count()) && (data_type_traits::is_floating_point(in_dt) || data_type_traits::is_floating_point(out_dt)) && in_dt != out_dt) { impl_candidate = impl_types::ocl; @@ -1482,7 +1482,7 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format for (auto& fused_op : node.get_fused_primitives()) { if (fused_op.node->is_type() && fused_op.deps.size() == 1) { auto eltw_in_layout = node.get_dependency(fused_op.dep_start_idx).get_output_layout(); - if (fused_op.node->as().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout)) { + if (program_helpers::needs_onednn_sum_post_op(fused_op.node->as(), eltw_in_layout)) { impl_candidate = impl_types::ocl; break; } @@ -1522,7 +1522,7 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format auto out_dt = out_layout.data_type; if ((out_layout.count() == in_layout.count()) && (data_type_traits::is_floating_point(in_dt) || data_type_traits::is_floating_point(out_dt)) && in_dt != out_dt && - fo.node->as().get_primitive()->needs_onednn_sum_post_op(in_layout)) { + program_helpers::needs_onednn_sum_post_op(fo.node->as(), in_layout)) { impl_candidate = impl_types::ocl; break; } diff --git a/src/plugins/intel_gpu/src/graph/network.cpp b/src/plugins/intel_gpu/src/graph/network.cpp index 63c5acafe96..b4fb14c6bc6 100644 --- a/src/plugins/intel_gpu/src/graph/network.cpp +++ b/src/plugins/intel_gpu/src/graph/network.cpp @@ -545,7 +545,7 @@ void network::allocate_primitives() { auto eltw_in_layout = eltw_in.get_output_layout(); auto out_layout = node->get_output_layout(); - if (!fused_op.node->as().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout)) + if (!program_helpers::needs_onednn_sum_post_op(fused_op.node->as(), eltw_in_layout)) continue; if (program_helpers::are_layouts_identical_for_onednn_sum_post_op(eltw_in_layout, out_layout)) { @@ -569,7 +569,7 @@ void network::allocate_primitives() { } } - if (fused_op.node->as().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout) && !can_reuse_eltwise_mem) { + if (program_helpers::needs_onednn_sum_post_op(fused_op.node->as(), eltw_in_layout) && !can_reuse_eltwise_mem) { throw std::runtime_error("Buffer reuse is required for onednn sum post operation."); } } diff --git a/src/plugins/intel_gpu/src/graph/program_helpers.cpp b/src/plugins/intel_gpu/src/graph/program_helpers.cpp index a680d357713..8cfa0ca5e39 100644 --- a/src/plugins/intel_gpu/src/graph/program_helpers.cpp +++ b/src/plugins/intel_gpu/src/graph/program_helpers.cpp @@ -191,4 +191,18 @@ bool program_helpers::are_layouts_identical_for_onednn_sum_post_op(layout input_ return false; } +bool program_helpers::needs_onednn_sum_post_op(const eltwise_node& n, layout input_layout) { + auto output_layout = n.get_output_layout(); + if (n.get_primitive()->mode == eltwise_mode::sum && + (input_layout.size.spatial[0] > 1 || input_layout.size.spatial[1] > 1 || input_layout.size.batch[0] > 1) + && output_layout.data_type == input_layout.data_type) { + return true; + } + + return false; +} + + + + } // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/program_node.cpp b/src/plugins/intel_gpu/src/graph/program_node.cpp index 22b0b88a3bc..22f27f5f1e5 100644 --- a/src/plugins/intel_gpu/src/graph/program_node.cpp +++ b/src/plugins/intel_gpu/src/graph/program_node.cpp @@ -4,6 +4,7 @@ #include "program_node.h" #include "intel_gpu/graph/program.hpp" +#include "program_helpers.h" #include "primitive_inst.h" #ifdef ENABLE_ONEDNN_FOR_GPU @@ -806,11 +807,11 @@ void program_node::init_onednn_primitive_attributes() { auto in = get_dependency(dep_idx).get_output_layout(); if (e_node.get_primitive()->mode == eltwise_mode::sum) { - if (e_node.get_primitive()->needs_onednn_sum_post_op(in)) { + if (program_helpers::needs_onednn_sum_post_op(e_node, in)) { post_ops.append_sum(1.0f, onednn::convert_data_type(in.data_type)); update_onednn_post_op_list(onednn_post_op_type::sum, dep_idx); } else { - dnnl::memory::desc in_desc = onednn::layout_to_memory_desc(in, dnnl::memory::format_tag::ab, true); + dnnl::memory::desc in_desc = onednn::layout_to_memory_desc(in); post_ops.append_binary(dnnl::algorithm::binary_add, in_desc); update_onednn_post_op_list(onednn_post_op_type::binary_add, dep_idx); } diff --git a/src/plugins/intel_gpu/tests/fusions/convolution_fusion_test.cpp b/src/plugins/intel_gpu/tests/fusions/convolution_fusion_test.cpp index e0114e8f27c..562cfa7c15d 100644 --- a/src/plugins/intel_gpu/tests/fusions/convolution_fusion_test.cpp +++ b/src/plugins/intel_gpu/tests/fusions/convolution_fusion_test.cpp @@ -3431,4 +3431,68 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_input_range, ::test convolution_test_params{ CASE_CONV_S8S8_15, 2, 3 }, })); + +// input:b_fs_yx_fsv32:u8 X weight:bfyx:i8 + eltwise_sum:b_fs_yx_fsv32:u8 +// After optimization: eltwise_any + binary_add +// DNNL_VERBOSE log with optimization: attr-post-ops:eltwise_tanh+binary_add:u8:14:aBcd32b+eltwise_linear:1 +class post_ops_optimizations_onednn_binary_add_full_tensor : public WeightsPrimitiveFusingTestOneDNN {}; +TEST_P(post_ops_optimizations_onednn_binary_add_full_tensor, basic) { + auto p = GetParam(); + create_topologies( + input_layout("input", get_input_layout(p)), + data("weights", get_mem(get_weights_layout(p))), + data("bias", get_mem(get_bias_layout(p))), + data("in_lo", get_mem(get_single_element_layout(p), 0)), + data("in_hi", get_mem(get_single_element_layout(p), 255)), + data("out_lo", get_mem(get_single_element_layout(p), 0)), + data("out_hi", get_mem(get_single_element_layout(p), 255)), + data("eltwise_data", get_mem(get_output_layout(p), 0, 255)), + convolution("conv_prim", "input", { "weights" }, { "bias" }, p.groups, p.stride, p.pad, p.dilation), + activation("activation", "conv_prim", activation_func::hyperbolic_tan), + eltwise("sum", { "activation", "eltwise_data" }, eltwise_mode::sum), + quantize("quantize", "sum", "in_lo", "in_hi", "out_lo", "out_hi", 256, data_types::u8), + reorder("reorder_bfyx", "quantize", p.default_format, data_types::f32) + ); + + tolerance = 1.f; + execute(p); +} + +// in_shape; out_shape; kernel; stride; pad; dilation; groups; data_type; input_format; weights_type; weights_format; default_type; default_format; +#define CASE_CONV_U8S8_FT_BINARY_ADD_1 { 1, 32, 4, 4 }, { 1, 16, 4, 4 }, { 1, 1, 3, 3 }, tensor{ 1 }, tensor{ 0, 0, 1, 1, 0 }, tensor{ 1 }, 1, data_types::u8, format::b_fs_yx_fsv32, data_types::i8, format::bfyx, data_types::f32, format::bfyx + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_binary_add_full_tensor, ::testing::ValuesIn(std::vector{ + // cases with batch = 1 + convolution_test_params{ CASE_CONV_U8S8_FT_BINARY_ADD_1, 2, 5 }, +})); + + +// input:b_fs_yx_fsv16:f16 X weight:bfyx:f16 + eltwise_sum:b_fs_yx_fsv16:f16 +// After optimization: eltwise_any + sum +// DNNL_VERBOSE log with optimization: attr-post-ops:eltwise_tanh+sum:1:0:f16 +class post_ops_optimizations_onednn_sum_full_tensor : public WeightsPrimitiveFusingTestOneDNN {}; +TEST_P(post_ops_optimizations_onednn_sum_full_tensor, basic) { + auto p = GetParam(); + create_topologies( + input_layout("input", get_input_layout(p)), + data("weights", get_mem(get_weights_layout(p))), + data("bias", get_mem(get_bias_layout(p))), + convolution("conv_prim", "input", { "weights" }, { "bias" }, p.groups, p.stride, p.pad, p.dilation), + activation("activation", "conv_prim", activation_func::hyperbolic_tan), + data("eltwise_data", get_mem(get_output_layout(p), 0, 255)), + eltwise("sum", { "activation", "eltwise_data" }, eltwise_mode::sum), + reorder("reorder_bfyx", "sum", p.default_format, data_types::f32) + ); + + tolerance = 1.f; + execute(p); +} + +#define CASE_CONV_F16F16_FT_ELTW_SUM_1 { 1, 32, 4, 4 }, { 1, 16, 4, 4 }, { 1, 1, 3, 3 }, tensor{ 1 }, tensor{ 0, 0, 1, 1, 0 }, tensor{ 1 }, 1, data_types::f16, format::b_fs_yx_fsv16, data_types::f16, format::bfyx, data_types::f32, format::bfyx + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_sum_full_tensor, ::testing::ValuesIn(std::vector{ + // cases with batch = 1 + convolution_test_params{ CASE_CONV_F16F16_FT_ELTW_SUM_1, 2, 4 }, +})); + #endif // ENABLE_ONEDNN_FOR_GPU