natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GPU] Binary post-op support for full tensor. (#9856) 

* [GPU] Binary post-op support for full tensor.

* Add unit tests

* Add a reorder if output dtype of conv layer is fp32.
This commit is contained in:
Jade Cho 2022-02-11 11:33:31 +09:00 committed by GitHub
parent fa69ee9596
commit dedcbeafa8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 136 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/plugins/intel_gpu/include/intel_gpu/primitives/eltwise.hpp
View File

@ -159,15 +159,6 @@ struct eltwise : public primitive_base<eltwise> {
}
}
bool needs_onednn_sum_post_op(layout input_layout) const {
if (mode == eltwise_mode::sum &&
(input_layout.size.spatial[0] > 1 || input_layout.size.spatial[1] > 1 || input_layout.size.batch[0] > 1)) {
return true;
}
return false;
}
/// @param mode Eltwise mode.
eltwise_mode mode;
/// @param coefficients Blob-wise coefficient for SUM operation.

2
src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_buffer_fusing.cpp
View File

@ -98,7 +98,7 @@ bool concat_in_place_optimization::match(concatenation_node& node) {
auto eltw_in_layout = eltw_in.get_output_layout();
auto out_layout = input->get_output_layout();
if (!fused_op.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout))
if (!program_helpers::needs_onednn_sum_post_op(fused_op.node->as<eltwise>(), eltw_in_layout))
continue;
if (program_helpers::are_layouts_identical_for_onednn_sum_post_op(eltw_in_layout, out_layout))
return false;

43
src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp
View File

@ -636,6 +636,49 @@ void reorder_inputs::run(program& p, layout_optimizer& lo, reorder_factory& rf)
conv_node.get_dependencies().front()->set_output_layout(new_layout, false);
}
}
// When the conv node is of onednn impl type and eltwise sum with full tensor is fused,
// changes the input format of eltwise sum post-op to use binary add.
if (conv_node.get_preferred_impl_type() == impl_types::onednn) {
std::vector<size_t> eltw_sum_dep_indices;
for (size_t i = 1; i < conv_node.get_dependencies().size(); i++) {
auto& dep = conv_node.get_dependency(i);
for (auto& fused_op : conv_node.get_fused_primitives()) {
if (fused_op.node->is_type<eltwise>()
&& fused_op.node->as<eltwise>().get_primitive()->mode == eltwise_mode::sum
&& !program_helpers::needs_onednn_sum_post_op(fused_op.node->as<eltwise>(),
conv_node.get_dependency(fused_op.dep_start_idx).get_output_layout())
&& conv_node.get_dependency(fused_op.dep_start_idx).get_users().size() == 1
&& conv_node.get_dependency(fused_op.dep_start_idx).id() == dep.id()) {
eltw_sum_dep_indices.push_back(i);
}
}
}
auto conv_layout = conv_node.get_output_layout();
for (auto& dep_id : eltw_sum_dep_indices) {
auto& prev_node = conv_node.get_dependency(dep_id);
auto old_layout = prev_node.get_output_layout();
auto expected_format = format::any;
if ((conv_layout.data_type == data_types::f16 || conv_layout.data_type == data_types::f32)
&& data_type_traits::is_i8_u8(old_layout.data_type)) {
if (conv_layout.format == format::b_fs_yx_fsv16)
expected_format = format::b_fs_yx_fsv32;
if (conv_layout.format == format::bs_fs_yx_bsv32_fsv16)
expected_format = format::bs_fs_yx_bsv32_fsv32;
}
if (expected_format != format::any && old_layout.format != expected_format) {
auto new_layout = old_layout;
new_layout.format = expected_format;
auto new_input = rf.get_reorder(prev_node.id(), old_layout, new_layout);
if (new_input.first) {
p.add_intermediate(new_input.first, conv_node, dep_id, !new_input.second);
}
conv_node.get_dependency(dep_id).set_output_layout(new_layout, false);
}
}
}
};
const auto reorder_input_fully_connected = [&p, &lo, &rf](typed_program_node<fully_connected>& fc_node) {

3
src/plugins/intel_gpu/src/graph/include/program_helpers.h
View File

@ -10,6 +10,7 @@
#include "intel_gpu/runtime/engine.hpp"
#include "intel_gpu/graph/program.hpp"
#include "data_inst.h"
#include "eltwise_inst.h"
#include <string>
#include <vector>
@ -126,6 +127,8 @@ struct program_helpers {
static layout get_weights_layout(typed_program_node<cldnn::data>& data_node, int32_t split);
static bool are_layouts_identical_for_onednn_sum_post_op(layout input_layout, layout output_layout);
static bool needs_onednn_sum_post_op(const eltwise_node& n, layout input_layout);
};
// Base class for performing pattern match style optimizations.

12
src/plugins/intel_gpu/src/graph/layout_optimizer.cpp
View File

@ -201,7 +201,7 @@ bool layout_optimizer::can_fuse_reorder(program_node& prev, program_node& next,
if (fused_op.node->is_type<eltwise>() && fused_op.deps.size() == 1) {
auto eltw_in_layout = next.get_dependency(fused_op.dep_start_idx).get_output_layout();
auto out_layout = next.get_output_layout();
if (fused_op.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout) &&
if (program_helpers::needs_onednn_sum_post_op(fused_op.node->as<eltwise>(), eltw_in_layout) &&
program_helpers::are_layouts_identical_for_onednn_sum_post_op(eltw_in_layout, out_layout) &&
prev.get_output_layout().format != out_layout.format)
return false;
@ -959,7 +959,7 @@ layout layout_optimizer::get_expected_layout(layout const& current_layout,
auto out_dt = out_layout.data_type;
if ((out_layout.count() == in_layout.count()) &&
(data_type_traits::is_floating_point(in_dt) || data_type_traits::is_floating_point(out_dt)) && in_dt != out_dt &&
fo.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(in_layout)) {
program_helpers::needs_onednn_sum_post_op(fo.node->as<eltwise>(), in_layout)) {
onednn_valid_post_ops = false;
break;
}
@ -1394,7 +1394,7 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format
if (fused_op.node->is_type<eltwise>() && node.get_dependencies().size() > fused_op.dep_start_idx && fused_op.deps.size() == 1) {
auto& eltw_in = node.get_dependency(fused_op.dep_start_idx);
if (program_helpers::are_layouts_identical_for_onednn_sum_post_op(eltw_in.get_output_layout(), node.get_output_layout()) &&
fused_op.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(eltw_in.get_output_layout())) {
program_helpers::needs_onednn_sum_post_op(fused_op.node->as<eltwise>(), eltw_in.get_output_layout())) {
if (sum_post_op_cnt > 0)
return impl_types::ocl;
@ -1448,7 +1448,7 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format
auto out_layout = node.get_output_layout();
auto in_dt = in_layout.data_type;
auto out_dt = out_layout.data_type;
if (fo.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(in_layout)) {
if (program_helpers::needs_onednn_sum_post_op(fo.node->as<eltwise>(), in_layout)) {
if ((out_layout.count() == in_layout.count()) &&
(data_type_traits::is_floating_point(in_dt) || data_type_traits::is_floating_point(out_dt)) && in_dt != out_dt) {
impl_candidate = impl_types::ocl;
@ -1482,7 +1482,7 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format
for (auto& fused_op : node.get_fused_primitives()) {
if (fused_op.node->is_type<eltwise>() && fused_op.deps.size() == 1) {
auto eltw_in_layout = node.get_dependency(fused_op.dep_start_idx).get_output_layout();
if (fused_op.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout)) {
if (program_helpers::needs_onednn_sum_post_op(fused_op.node->as<eltwise>(), eltw_in_layout)) {
impl_candidate = impl_types::ocl;
break;
}
@ -1522,7 +1522,7 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format
auto out_dt = out_layout.data_type;
if ((out_layout.count() == in_layout.count()) &&
(data_type_traits::is_floating_point(in_dt) || data_type_traits::is_floating_point(out_dt)) && in_dt != out_dt &&
fo.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(in_layout)) {
program_helpers::needs_onednn_sum_post_op(fo.node->as<eltwise>(), in_layout)) {
impl_candidate = impl_types::ocl;
break;
}

4
src/plugins/intel_gpu/src/graph/network.cpp
View File

@ -545,7 +545,7 @@ void network::allocate_primitives() {
auto eltw_in_layout = eltw_in.get_output_layout();
auto out_layout = node->get_output_layout();
if (!fused_op.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout))
if (!program_helpers::needs_onednn_sum_post_op(fused_op.node->as<eltwise>(), eltw_in_layout))
continue;
if (program_helpers::are_layouts_identical_for_onednn_sum_post_op(eltw_in_layout, out_layout)) {
@ -569,7 +569,7 @@ void network::allocate_primitives() {
}
}
if (fused_op.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout) && !can_reuse_eltwise_mem) {
if (program_helpers::needs_onednn_sum_post_op(fused_op.node->as<eltwise>(), eltw_in_layout) && !can_reuse_eltwise_mem) {
throw std::runtime_error("Buffer reuse is required for onednn sum post operation.");
}
}

14
src/plugins/intel_gpu/src/graph/program_helpers.cpp
View File

@ -191,4 +191,18 @@ bool program_helpers::are_layouts_identical_for_onednn_sum_post_op(layout input_
return false;
}
bool program_helpers::needs_onednn_sum_post_op(const eltwise_node& n, layout input_layout) {
auto output_layout = n.get_output_layout();
if (n.get_primitive()->mode == eltwise_mode::sum &&
(input_layout.size.spatial[0] > 1 || input_layout.size.spatial[1] > 1 || input_layout.size.batch[0] > 1)
&& output_layout.data_type == input_layout.data_type) {
return true;
}
return false;
}
} // namespace cldnn

5
src/plugins/intel_gpu/src/graph/program_node.cpp
View File

@ -4,6 +4,7 @@
#include "program_node.h"
#include "intel_gpu/graph/program.hpp"
#include "program_helpers.h"
#include "primitive_inst.h"
#ifdef ENABLE_ONEDNN_FOR_GPU
@ -806,11 +807,11 @@ void program_node::init_onednn_primitive_attributes() {
auto in = get_dependency(dep_idx).get_output_layout();
if (e_node.get_primitive()->mode == eltwise_mode::sum) {
if (e_node.get_primitive()->needs_onednn_sum_post_op(in)) {
if (program_helpers::needs_onednn_sum_post_op(e_node, in)) {
post_ops.append_sum(1.0f, onednn::convert_data_type(in.data_type));
update_onednn_post_op_list(onednn_post_op_type::sum, dep_idx);
} else {
dnnl::memory::desc in_desc = onednn::layout_to_memory_desc(in, dnnl::memory::format_tag::ab, true);
dnnl::memory::desc in_desc = onednn::layout_to_memory_desc(in);
post_ops.append_binary(dnnl::algorithm::binary_add, in_desc);
update_onednn_post_op_list(onednn_post_op_type::binary_add, dep_idx);
}

64
src/plugins/intel_gpu/tests/fusions/convolution_fusion_test.cpp
View File

@ -3431,4 +3431,68 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_input_range, ::test
convolution_test_params{ CASE_CONV_S8S8_15, 2, 3 },
}));
// input:b_fs_yx_fsv32:u8 X weight:bfyx:i8 + eltwise_sum:b_fs_yx_fsv32:u8
// After optimization: eltwise_any + binary_add
// DNNL_VERBOSE log with optimization: attr-post-ops:eltwise_tanh+binary_add:u8:14:aBcd32b+eltwise_linear:1
class post_ops_optimizations_onednn_binary_add_full_tensor : public WeightsPrimitiveFusingTestOneDNN {};
TEST_P(post_ops_optimizations_onednn_binary_add_full_tensor, basic) {
auto p = GetParam();
create_topologies(
input_layout("input", get_input_layout(p)),
data("weights", get_mem(get_weights_layout(p))),
data("bias", get_mem(get_bias_layout(p))),
data("in_lo", get_mem(get_single_element_layout(p), 0)),
data("in_hi", get_mem(get_single_element_layout(p), 255)),
data("out_lo", get_mem(get_single_element_layout(p), 0)),
data("out_hi", get_mem(get_single_element_layout(p), 255)),
data("eltwise_data", get_mem(get_output_layout(p), 0, 255)),
convolution("conv_prim", "input", { "weights" }, { "bias" }, p.groups, p.stride, p.pad, p.dilation),
activation("activation", "conv_prim", activation_func::hyperbolic_tan),
eltwise("sum", { "activation", "eltwise_data" }, eltwise_mode::sum),
quantize("quantize", "sum", "in_lo", "in_hi", "out_lo", "out_hi", 256, data_types::u8),
reorder("reorder_bfyx", "quantize", p.default_format, data_types::f32)
);
tolerance = 1.f;
execute(p);
}
// in_shape; out_shape; kernel; stride; pad; dilation; groups; data_type; input_format; weights_type; weights_format; default_type; default_format;
#define CASE_CONV_U8S8_FT_BINARY_ADD_1 { 1, 32, 4, 4 }, { 1, 16, 4, 4 }, { 1, 1, 3, 3 }, tensor{ 1 }, tensor{ 0, 0, 1, 1, 0 }, tensor{ 1 }, 1, data_types::u8, format::b_fs_yx_fsv32, data_types::i8, format::bfyx, data_types::f32, format::bfyx
INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_binary_add_full_tensor, ::testing::ValuesIn(std::vector<convolution_test_params>{
// cases with batch = 1
convolution_test_params{ CASE_CONV_U8S8_FT_BINARY_ADD_1, 2, 5 },
}));
// input:b_fs_yx_fsv16:f16 X weight:bfyx:f16 + eltwise_sum:b_fs_yx_fsv16:f16
// After optimization: eltwise_any + sum
// DNNL_VERBOSE log with optimization: attr-post-ops:eltwise_tanh+sum:1:0:f16
class post_ops_optimizations_onednn_sum_full_tensor : public WeightsPrimitiveFusingTestOneDNN {};
TEST_P(post_ops_optimizations_onednn_sum_full_tensor, basic) {
auto p = GetParam();
create_topologies(
input_layout("input", get_input_layout(p)),
data("weights", get_mem(get_weights_layout(p))),
data("bias", get_mem(get_bias_layout(p))),
convolution("conv_prim", "input", { "weights" }, { "bias" }, p.groups, p.stride, p.pad, p.dilation),
activation("activation", "conv_prim", activation_func::hyperbolic_tan),
data("eltwise_data", get_mem(get_output_layout(p), 0, 255)),
eltwise("sum", { "activation", "eltwise_data" }, eltwise_mode::sum),
reorder("reorder_bfyx", "sum", p.default_format, data_types::f32)
);
tolerance = 1.f;
execute(p);
}
#define CASE_CONV_F16F16_FT_ELTW_SUM_1 { 1, 32, 4, 4 }, { 1, 16, 4, 4 }, { 1, 1, 3, 3 }, tensor{ 1 }, tensor{ 0, 0, 1, 1, 0 }, tensor{ 1 }, 1, data_types::f16, format::b_fs_yx_fsv16, data_types::f16, format::bfyx, data_types::f32, format::bfyx
INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_sum_full_tensor, ::testing::ValuesIn(std::vector<convolution_test_params>{
// cases with batch = 1
convolution_test_params{ CASE_CONV_F16F16_FT_ELTW_SUM_1, 2, 4 },
}));
#endif // ENABLE_ONEDNN_FOR_GPU