natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GPU] Fix decompression scales reorder when data is reused (#24842) 

### Details:
- In case when scale tensor was used by multiple compressed FCs we had
an exception on attempt to insert reorder between them. This patch
changes reorder name so now multiple reorders are inserted for each
Scale+FC pair, and then on later stages we merge them if they do same
tensor transform.
This commit is contained in:
Vladimir Paramuzov 2024-06-05 10:10:37 +04:00 committed by GitHub
parent adb590d5f4
commit f952f73d6b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 106 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_quantization.cpp
View File

@ -503,7 +503,7 @@ static void optimize_weights_decompression_parameters(fully_connected_node& fc_n
auto& dep = fc_node.get_dependency(dep_id);
auto target_layout = dep.get_output_layout();
target_layout.format = format::fbyx;
auto reorder_prim = std::make_shared<reorder>(dep.id() + "_reorder", dep.id(), target_layout);
auto reorder_prim = std::make_shared<reorder>(dep.id() + "_reorder_" + fc_node.id(), dep.id(), target_layout);
p.add_intermediate(reorder_prim, fc_node, dep_id, true);
fc_node.get_dependency(dep_id).recalc_output_layout(false);
};

105
src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp
View File

@ -1343,6 +1343,107 @@ public:
ASSERT_NEAR(output_ptr_ref[i], output_ptr[i], 9.0) << "i = " << i;
}
void test_compressed_int4_scale_reuse(bool is_caching_test, bool is_dynamic, long int batch_num, long int scales_group_size = 128) {
tests::random_generator rg(GET_SUITE_NAME);
auto& engine = get_test_engine();
long int ifm_num = 256;
long int ofm_num = 256;
auto input_mem = engine.allocate_memory({ { batch_num, ifm_num}, data_types::f16, format::bfyx });
auto weights_mem1 = engine.allocate_memory({ {ofm_num, ifm_num}, data_types::u4, format::bfyx });
auto weights_mem2 = engine.allocate_memory({ {ofm_num, ifm_num}, data_types::u4, format::bfyx });
auto scale_mem = engine.allocate_memory({ {ofm_num, ifm_num / scales_group_size}, data_types::f16, format::bfyx });
auto input_data = rg.generate_random_1d<ov::float16>(batch_num * ifm_num, -2.0f, 2.0f);
set_values(input_mem, input_data);
auto weigths_data = rg.generate_random_1d<uint8_t>(ofm_num * ifm_num / 2, 0, 10);
set_values(weights_mem1, weigths_data);
set_values(weights_mem2, weigths_data);
auto scale_data = rg.generate_random_1d<ov::float16>(ofm_num * ifm_num / scales_group_size, -4.0f, 4.0f);
set_values(scale_mem, scale_data);
auto in_layout = is_dynamic ? layout{ {-1, ifm_num}, data_types::f16, format::bfyx }
: layout{ {batch_num, ifm_num}, data_types::f16, format::bfyx };
auto fc_prim1 = fully_connected("fc_prim1", input_info("input"), "weights1", "", "scale", "", data_types::f16, padding(), 2, 2);
auto fc_prim2 = fully_connected("fc_prim2", input_info("input"), "weights2", "", "scale", "", data_types::f16, padding(), 2, 2);
fc_prim1.decompression_zero_point_scalar = 8;
fc_prim2.decompression_zero_point_scalar = 8;
auto get_ref_results = [&]() {
topology topology(
input_layout("input", in_layout),
data("weights1", weights_mem1),
data("weights2", weights_mem2),
data("scale", scale_mem),
fc_prim1,
fc_prim2
);
auto config = get_test_default_config(engine);
config.set_property(ov::intel_gpu::allow_new_shape_infer(true));
network network(engine, topology, config);
network.set_input_data("input", input_mem);
auto outputs = network.execute();
OPENVINO_ASSERT(outputs.size() == 2);
std::vector<cldnn::memory::ptr> res {
engine.reinterpret_buffer(*outputs.at("fc_prim1").get_memory(), outputs.at("fc_prim1").get_layout()),
engine.reinterpret_buffer(*outputs.at("fc_prim2").get_memory(), outputs.at("fc_prim2").get_layout()),
};
return res;
};
topology topology(
input_layout("input", in_layout),
data("weights1", weights_mem1),
data("weights2", weights_mem2),
data("scale", scale_mem),
fc_prim1,
fc_prim2
);
auto config = get_test_default_config(engine);
config.set_property(ov::intel_gpu::allow_new_shape_infer(true));
config.set_property(ov::intel_gpu::optimize_data(true));
network::ptr network = get_network(engine, topology, config, get_test_stream_ptr(), is_caching_test);
// Impl is selected only when it is running from cldnn
if (is_dynamic && !engine.get_device_info().supports_immad) {
auto inst = network->get_primitive("fc_prim1");
auto impl = inst->get_impl();
ASSERT_TRUE(impl != NULL);
ASSERT_EQ(impl->get_kernels().size(), 2);
}
network->set_input_data("input", input_mem);
auto outputs = network->execute();
ASSERT_EQ(outputs.size(), size_t(2));
std::vector<cldnn::memory::ptr> output_mem {
engine.reinterpret_buffer(*outputs.at("fc_prim1").get_memory(), outputs.at("fc_prim1").get_layout()),
engine.reinterpret_buffer(*outputs.at("fc_prim2").get_memory(), outputs.at("fc_prim2").get_layout()),
};
auto ref_output_mem = get_ref_results();
for (size_t i = 0; i < 2; i++) {
cldnn::mem_lock<ov::float16> output_ptr (output_mem[i], get_test_stream());
cldnn::mem_lock<ov::float16> output_ptr_ref (ref_output_mem[i], get_test_stream());
for (size_t i = 0; i < output_ptr_ref.size(); i++)
ASSERT_NEAR(output_ptr_ref[i], output_ptr[i], 9.0) << "i = " << i;
}
}
void test_compressed_int8_scale_zp_bias(bool is_caching_test) {
auto& engine = get_test_engine();
@ -3013,6 +3114,10 @@ TEST_F(fully_connected_gpu_tests, compressed_int4_scale) {
this->test_compressed_int4_scale(false, false, 256);
}
TEST_F(fully_connected_gpu_tests, compressed_int4_reuse_scale) {
this->test_compressed_int4_scale_reuse(false, true, 256);
}
TEST_F(fully_connected_gpu_tests, compressed_int4_scale_cached) {
this->test_compressed_int4_scale(true, false, 256);
}