From f952f73d6baeed66772fd25a3b8c04423f2fa560 Mon Sep 17 00:00:00 2001 From: Vladimir Paramuzov Date: Wed, 5 Jun 2024 10:10:37 +0400 Subject: [PATCH] [GPU] Fix decompression scales reorder when data is reused (#24842) ### Details: - In case when scale tensor was used by multiple compressed FCs we had an exception on attempt to insert reorder between them. This patch changes reorder name so now multiple reorders are inserted for each Scale+FC pair, and then on later stages we merge them if they do same tensor transform. --- .../graph_optimizer/prepare_quantization.cpp | 2 +- .../test_cases/fully_connected_gpu_test.cpp | 105 ++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_quantization.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_quantization.cpp index b41b4457ecb..c17490488b6 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_quantization.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_quantization.cpp @@ -503,7 +503,7 @@ static void optimize_weights_decompression_parameters(fully_connected_node& fc_n auto& dep = fc_node.get_dependency(dep_id); auto target_layout = dep.get_output_layout(); target_layout.format = format::fbyx; - auto reorder_prim = std::make_shared(dep.id() + "_reorder", dep.id(), target_layout); + auto reorder_prim = std::make_shared(dep.id() + "_reorder_" + fc_node.id(), dep.id(), target_layout); p.add_intermediate(reorder_prim, fc_node, dep_id, true); fc_node.get_dependency(dep_id).recalc_output_layout(false); }; diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp index 943515924ed..34ef987a31f 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/fully_connected_gpu_test.cpp @@ -1343,6 +1343,107 @@ public: ASSERT_NEAR(output_ptr_ref[i], output_ptr[i], 9.0) << "i = " << i; } + void test_compressed_int4_scale_reuse(bool is_caching_test, bool is_dynamic, long int batch_num, long int scales_group_size = 128) { + tests::random_generator rg(GET_SUITE_NAME); + auto& engine = get_test_engine(); + + long int ifm_num = 256; + long int ofm_num = 256; + + auto input_mem = engine.allocate_memory({ { batch_num, ifm_num}, data_types::f16, format::bfyx }); + auto weights_mem1 = engine.allocate_memory({ {ofm_num, ifm_num}, data_types::u4, format::bfyx }); + auto weights_mem2 = engine.allocate_memory({ {ofm_num, ifm_num}, data_types::u4, format::bfyx }); + auto scale_mem = engine.allocate_memory({ {ofm_num, ifm_num / scales_group_size}, data_types::f16, format::bfyx }); + + auto input_data = rg.generate_random_1d(batch_num * ifm_num, -2.0f, 2.0f); + set_values(input_mem, input_data); + + auto weigths_data = rg.generate_random_1d(ofm_num * ifm_num / 2, 0, 10); + set_values(weights_mem1, weigths_data); + set_values(weights_mem2, weigths_data); + + auto scale_data = rg.generate_random_1d(ofm_num * ifm_num / scales_group_size, -4.0f, 4.0f); + set_values(scale_mem, scale_data); + + auto in_layout = is_dynamic ? layout{ {-1, ifm_num}, data_types::f16, format::bfyx } + : layout{ {batch_num, ifm_num}, data_types::f16, format::bfyx }; + + auto fc_prim1 = fully_connected("fc_prim1", input_info("input"), "weights1", "", "scale", "", data_types::f16, padding(), 2, 2); + auto fc_prim2 = fully_connected("fc_prim2", input_info("input"), "weights2", "", "scale", "", data_types::f16, padding(), 2, 2); + + fc_prim1.decompression_zero_point_scalar = 8; + fc_prim2.decompression_zero_point_scalar = 8; + + auto get_ref_results = [&]() { + topology topology( + input_layout("input", in_layout), + data("weights1", weights_mem1), + data("weights2", weights_mem2), + data("scale", scale_mem), + fc_prim1, + fc_prim2 + ); + + auto config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + + network network(engine, topology, config); + network.set_input_data("input", input_mem); + + auto outputs = network.execute(); + OPENVINO_ASSERT(outputs.size() == 2); + + std::vector res { + engine.reinterpret_buffer(*outputs.at("fc_prim1").get_memory(), outputs.at("fc_prim1").get_layout()), + engine.reinterpret_buffer(*outputs.at("fc_prim2").get_memory(), outputs.at("fc_prim2").get_layout()), + }; + + return res; + }; + + topology topology( + input_layout("input", in_layout), + data("weights1", weights_mem1), + data("weights2", weights_mem2), + data("scale", scale_mem), + fc_prim1, + fc_prim2 + ); + + auto config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::intel_gpu::optimize_data(true)); + + network::ptr network = get_network(engine, topology, config, get_test_stream_ptr(), is_caching_test); + + // Impl is selected only when it is running from cldnn + if (is_dynamic && !engine.get_device_info().supports_immad) { + auto inst = network->get_primitive("fc_prim1"); + auto impl = inst->get_impl(); + ASSERT_TRUE(impl != NULL); + ASSERT_EQ(impl->get_kernels().size(), 2); + } + + network->set_input_data("input", input_mem); + + auto outputs = network->execute(); + ASSERT_EQ(outputs.size(), size_t(2)); + + std::vector output_mem { + engine.reinterpret_buffer(*outputs.at("fc_prim1").get_memory(), outputs.at("fc_prim1").get_layout()), + engine.reinterpret_buffer(*outputs.at("fc_prim2").get_memory(), outputs.at("fc_prim2").get_layout()), + }; + auto ref_output_mem = get_ref_results(); + + for (size_t i = 0; i < 2; i++) { + cldnn::mem_lock output_ptr (output_mem[i], get_test_stream()); + cldnn::mem_lock output_ptr_ref (ref_output_mem[i], get_test_stream()); + + for (size_t i = 0; i < output_ptr_ref.size(); i++) + ASSERT_NEAR(output_ptr_ref[i], output_ptr[i], 9.0) << "i = " << i; + } + } + void test_compressed_int8_scale_zp_bias(bool is_caching_test) { auto& engine = get_test_engine(); @@ -3013,6 +3114,10 @@ TEST_F(fully_connected_gpu_tests, compressed_int4_scale) { this->test_compressed_int4_scale(false, false, 256); } +TEST_F(fully_connected_gpu_tests, compressed_int4_reuse_scale) { + this->test_compressed_int4_scale_reuse(false, true, 256); +} + TEST_F(fully_connected_gpu_tests, compressed_int4_scale_cached) { this->test_compressed_int4_scale(true, false, 256); }