diff --git a/src/plugins/intel_cpu/src/nodes/gather.cpp b/src/plugins/intel_cpu/src/nodes/gather.cpp index e089707adda..434a32073ac 100644 --- a/src/plugins/intel_cpu/src/nodes/gather.cpp +++ b/src/plugins/intel_cpu/src/nodes/gather.cpp @@ -171,6 +171,7 @@ void Gather::initSupportedPrimitiveDescriptors() { } scale_group_size = getInputShapeAtPort(GATHER_DATA).getElementsCount() / getInputShapeAtPort(GATHER_SCALE).getElementsCount(); + have_scalar_scale = getInputShapeAtPort(GATHER_SCALE).getElementsCount() == 1u; if (getOriginalInputsNumber() == 5u) { ov::element::Type zpPrecision = getOriginalInputPrecisionAtPort(GATHER_ZP); @@ -179,6 +180,7 @@ void Gather::initSupportedPrimitiveDescriptors() { } have_zp = true; + have_scalar_zp = getInputShapeAtPort(GATHER_ZP).getElementsCount() == 1u; zp_group_size = getInputShapeAtPort(GATHER_DATA).getElementsCount() / getInputShapeAtPort(GATHER_ZP).getElementsCount(); addSupportedPrimDesc({{LayoutType::ncsp, dataPrecision}, @@ -630,7 +632,6 @@ void Gather::execCompressed4Bit() { const auto* scale = getSrcDataAtPortAs(GATHER_SCALE); const size_t dstAfterBatchSize = betweenBatchAndAxisSize * specIdxAndAfterAxSize; - parallel_for2d(beforeBatchSize, specIndicesSize, [&](const size_t b, const size_t j) { int ii = srcIndices[b * specIndicesSize + j]; if (ii < 0) { @@ -649,15 +650,46 @@ void Gather::execCompressed4Bit() { OUT_TYPE* pdst = &dstData[dstIdx]; - const size_t scale_offset = srcIdx / scale_group_size; - auto cur_zp = have_zp ? zp[srcIdx / zp_group_size] : 0; size_t p = srcIdx; size_t dst_idx = 0; - for (; p < srcIdx + afterAxisSize; p++) { - auto val = srcData[p >> 1]; - pdst[dst_idx] = static_cast((get4Bit(val, p % 2) - cur_zp) * scale[scale_offset]); - dst_idx++; + // heuristic: + // ((isAxisInputConst && axis == 0) && (cond1 || cond2)) take >99% probability + bool processed = false; + if (isAxisInputConst && axis == 0) { + bool cond1 = have_zp && zp_group_size == scale_group_size; + bool cond2 = (!have_zp) || have_scalar_zp; + bool cond3 = have_scalar_scale && cond2; + if (cond3) { + processed = true; + for (; p < srcIdx + afterAxisSize; p++) { + auto val = srcData[p >> 1]; + pdst[dst_idx] = static_cast((get4Bit(val, p % 2) - zp[0]) * scale[0]); + dst_idx++; + } + } else if (cond1 || cond2) { + processed = true; + for (; p < srcIdx + afterAxisSize; p += scale_group_size) { + const auto& cur_scale = scale[p / scale_group_size]; + const auto& cur_zp = cond2 ? zp[0] : zp[p / zp_group_size]; + for (size_t g = p; g < p + scale_group_size; g++) { + auto val = srcData[g >> 1]; + pdst[dst_idx] = static_cast((get4Bit(val, g % 2) - cur_zp) * cur_scale); + dst_idx++; + } + } + } + } + + // Reference + if (!processed) { + for (; p < srcIdx + afterAxisSize; p++) { + auto val = srcData[p >> 1]; + const size_t scale_offset = p / scale_group_size; + auto cur_zp = have_zp ? zp[p / zp_group_size] : 0; + pdst[dst_idx] = static_cast((get4Bit(val, p % 2) - cur_zp) * scale[scale_offset]); + dst_idx++; + } } } } else { @@ -699,13 +731,47 @@ void Gather::execCompressed8Bit() { size_t srcIdx = c1 + axisAndAfterAxisSize * i; size_t dstIdx = c2 + specIdxAndAfterAxSize * i; - const IN_TYPE* psrc = &srcData[srcIdx]; OUT_TYPE* pdst = &dstData[dstIdx]; - const size_t scale_offset = srcIdx / scale_group_size; - auto cur_zp = have_zp ? zp[srcIdx / zp_group_size] : 0; - for (size_t p = 0; p < afterAxisSize; p++) { - pdst[p] = static_cast((static_cast(psrc[p]) - cur_zp) * scale[scale_offset]); + size_t p = srcIdx; + size_t dst_idx = 0; + + // heuristic: + // ((isAxisInputConst && axis == 0) && (cond1 || cond2)) take >99% probability + bool processed = false; + if (isAxisInputConst && axis == 0) { + bool cond1 = have_zp && zp_group_size == scale_group_size; + bool cond2 = (!have_zp) || have_scalar_zp; + bool cond3 = have_scalar_scale && cond2; + if (cond3) { + processed = true; + for (; p < srcIdx + afterAxisSize; p++) { + pdst[dst_idx] = static_cast((static_cast(srcData[p]) - zp[0]) * scale[0]); + dst_idx++; + } + } else if (cond1 || cond2) { + processed = true; + for (; p < srcIdx + afterAxisSize; p += scale_group_size) { + const auto& cur_scale = scale[p / scale_group_size]; + const auto& cur_zp = cond2 ? zp[0] : zp[p / zp_group_size]; + for (size_t g = p; g < p + scale_group_size; g++) { + pdst[dst_idx] = + static_cast((static_cast(srcData[g]) - cur_zp) * cur_scale); + dst_idx++; + } + } + } + } + + // Reference + if (!processed) { + for (; p < srcIdx + afterAxisSize; p++) { + const size_t scale_offset = p / scale_group_size; + auto cur_zp = have_zp ? zp[p / zp_group_size] : 0; + pdst[dst_idx] = + static_cast((static_cast(srcData[p]) - cur_zp) * scale[scale_offset]); + dst_idx++; + } } } } else { diff --git a/src/plugins/intel_cpu/src/nodes/gather.h b/src/plugins/intel_cpu/src/nodes/gather.h index a9ec8727048..96dad228f65 100644 --- a/src/plugins/intel_cpu/src/nodes/gather.h +++ b/src/plugins/intel_cpu/src/nodes/gather.h @@ -106,6 +106,8 @@ private: static constexpr size_t GATHER_ZP = 4; bool have_zp = false; + bool have_scalar_zp = false; + bool have_scalar_scale = false; size_t zp_group_size = 1u; size_t scale_group_size = 1u; diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp index 6950415b784..59ce6edeef5 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp @@ -24,12 +24,14 @@ const std::vector weights_precisions = {ov::element::u8, const std::vector input_shapes_basic = { {{2, 5}, {{-1, -1}, {{2, 3}}}, 0, 0}, {{15, 32}, {{-1, -1}, {{2, 3}}}, 1, 0, 16}, + {{15, 32}, {{-1, -1}, {{2, 3}}}, 0, 0, 16}, {{2, 5}, {{}, {{2, 3}}}, 1, -1}, {{15, 16, 2}, {{-1, -1}, {{2, 3}}}, 0, 0}, }; const std::vector add_decompression_sub = {true, false}; const std::vector reshape_on_decompression = {true, false}; const std::vector per_tensor_zp = {true, false}; +const std::vector per_tensor_scale = {true, false}; INSTANTIATE_TEST_SUITE_P(smoke_GatherCompressedWeights_basic, GatherWeightsDecompression, @@ -39,7 +41,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_GatherCompressedWeights_basic, ::testing::ValuesIn(output_precisions), ::testing::ValuesIn(add_decompression_sub), ::testing::ValuesIn(reshape_on_decompression), - ::testing::ValuesIn(per_tensor_zp)), + ::testing::ValuesIn(per_tensor_zp), + ::testing::ValuesIn(per_tensor_scale)), GatherWeightsDecompression::get_test_case_name); } // namespace diff --git a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp index ca914231dad..e0281c5232c 100644 --- a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp @@ -22,12 +22,14 @@ const std::vector weights_precisions = {ov::element::u8, const std::vector input_shapes_basic = { {{2, 5}, {{-1, -1}, {{2, 3}}}, 1, 1}, {{15, 32}, {{-1, -1}, {{2, 3}}}, 1, 0, 16}, + {{15, 32}, {{-1, -1}, {{2, 3}}}, 0, 0, 16}, {{2, 5}, {{}, {{2, 3}}}, 1, -1}, {{15, 16, 2}, {{-1, -1}, {{2, 3}}}, 0, 0}, }; const std::vector add_decompression_sub = {true, false}; const std::vector reshape_on_decompression = {true, false}; const std::vector per_tensor_zp = {true, false}; +const std::vector per_tensor_scale = {true, false}; INSTANTIATE_TEST_SUITE_P(smoke_GatherCompressedWeights_basic, GatherWeightsDecompression, @@ -37,7 +39,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_GatherCompressedWeights_basic, ::testing::ValuesIn(output_precisions), ::testing::ValuesIn(add_decompression_sub), ::testing::ValuesIn(reshape_on_decompression), - ::testing::ValuesIn(per_tensor_zp)), + ::testing::ValuesIn(per_tensor_zp), + ::testing::ValuesIn(per_tensor_scale)), GatherWeightsDecompression::get_test_case_name); } // namespace diff --git a/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/gather_weights_decompression.hpp b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/gather_weights_decompression.hpp index 0bbf87912c7..5bfb4039cea 100644 --- a/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/gather_weights_decompression.hpp +++ b/src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/gather_weights_decompression.hpp @@ -52,6 +52,7 @@ using GatherWeightsDecompressionParams = std::tuple; // per-tensor zero-point class GatherWeightsDecompression : public testing::WithParamInterface, @@ -69,14 +70,16 @@ protected: const ov::element::Type output_precision, const bool add_subtract, const bool reshape_on_decompression, - const bool per_tensor_zp); + const bool per_tensor_zp, + const bool per_tensor_scale); std::shared_ptr init_compressed_weights_subgraph(const ov::Shape& data_shape, const int group_size, const ov::element::Type data_precision, const ov::element::Type output_precision, const bool add_subtract, const bool reshape_on_decompression_constant, - const bool per_tensor_zp); + const bool per_tensor_zp, + const bool per_tensor_scale); void generate_inputs(const std::vector& target_input_static_shapes) override; void check_results(); void SetUp() override; diff --git a/src/tests/functional/shared_test_classes/src/subgraph/gather_weights_decompression.cpp b/src/tests/functional/shared_test_classes/src/subgraph/gather_weights_decompression.cpp index b6f85140d15..ce238827f62 100644 --- a/src/tests/functional/shared_test_classes/src/subgraph/gather_weights_decompression.cpp +++ b/src/tests/functional/shared_test_classes/src/subgraph/gather_weights_decompression.cpp @@ -17,6 +17,7 @@ std::string GatherWeightsDecompression::get_test_case_name( bool decompression_sub; bool reshape_on_decompression; bool per_tensor_zp; + bool per_tensor_scale; std::tie(target_device, shape_params, @@ -24,7 +25,8 @@ std::string GatherWeightsDecompression::get_test_case_name( output_precision, decompression_sub, reshape_on_decompression, - per_tensor_zp) = obj.param; + per_tensor_zp, + per_tensor_scale) = obj.param; std::ostringstream result; result << "target_device=" << target_device << "_"; @@ -34,12 +36,15 @@ std::string GatherWeightsDecompression::get_test_case_name( for (const auto& actual_shape : shape_params.indices_shape.second) { result << ov::test::utils::partialShape2str({actual_shape}) << "_"; } + result << "axis=" << shape_params.axis << "_"; + result << "batch_dims=" << shape_params.batch_dims << "_"; result << "group_size=" << shape_params.decompression_group_size << "_"; result << "data_precision=" << data_precision << "_"; result << "output_precision=" << output_precision << "_"; result << "decompression_subtract=" << decompression_sub << "_"; result << "reshape_on_decompression=" << reshape_on_decompression << "_"; result << "per_tensor_zp=" << per_tensor_zp; + result << "per_tensor_scale=" << per_tensor_scale; return result.str(); } @@ -53,7 +58,8 @@ std::shared_ptr GatherWeightsDecompression::init_subgraph(const ov::S const ov::element::Type output_precision, const bool add_subtract, const bool reshape_on_decompression, - const bool per_tensor_zp) { + const bool per_tensor_zp, + const bool per_tensor_scale) { ov::ParameterVector params{std::make_shared(ov::element::i32, indices_shape)}; auto axis_const = ov::op::v0::Constant::create(ov::element::i32, {1}, {axis}); const auto data_subgraph = init_compressed_weights_subgraph(data_shape, @@ -62,7 +68,8 @@ std::shared_ptr GatherWeightsDecompression::init_subgraph(const ov::S output_precision, add_subtract, reshape_on_decompression, - per_tensor_zp); + per_tensor_zp, + per_tensor_scale); auto gather = std::make_shared(data_subgraph, params[0], axis_const, batch_dims); gather->set_friendly_name("gather_node"); @@ -75,7 +82,8 @@ std::shared_ptr GatherWeightsDecompression::init_compressed_weights_su const ov::element::Type output_precision, const bool add_subtract, const bool reshape_on_decompression_constant, - const bool per_tensor_zp) { + const bool per_tensor_zp, + const bool per_tensor_scale) { const bool group_decompression = group_size != -1; // Weights has shape [I, D], where // I - index @@ -141,7 +149,8 @@ std::shared_ptr GatherWeightsDecompression::init_compressed_weights_su in_data.start_from = -0.5; in_data.range = 1; in_data.resolution = 30000; - auto scale_tensor = ov::test::utils::create_and_fill_tensor(output_precision, scaleshift_const_shape, in_data); + auto shift_tensor_shape = per_tensor_scale ? ov::Shape{1} : scaleshift_const_shape; + auto scale_tensor = ov::test::utils::create_and_fill_tensor(output_precision, shift_tensor_shape, in_data); for (size_t i = 0; i < scale_tensor.get_size(); i++) { if (output_precision == ov::element::f16) scale_tensor.data()[i] /= ov::float16(16.f); @@ -149,7 +158,7 @@ std::shared_ptr GatherWeightsDecompression::init_compressed_weights_su scale_tensor.data()[i] /= 16.f; } std::shared_ptr scale_const = std::make_shared(scale_tensor); - if (reshape_on_decompression_constant) { + if (reshape_on_decompression_constant && !per_tensor_scale) { auto scale_reshape_const = ov::op::v0::Constant::create(ov::element::i32, {scaleshift_target_shape.size()}, scaleshift_target_shape); auto scale_reshape = std::make_shared(scale_const, scale_reshape_const, false); @@ -204,6 +213,7 @@ void GatherWeightsDecompression::SetUp() { bool decompression_sub; bool reshape_on_decompression; bool per_tensor_zp; + bool per_tensor_scale; std::tie(targetDevice, shape_params, @@ -211,7 +221,8 @@ void GatherWeightsDecompression::SetUp() { output_precision, decompression_sub, reshape_on_decompression, - per_tensor_zp) = GetParam(); + per_tensor_zp, + per_tensor_scale) = GetParam(); init_input_shapes({shape_params.indices_shape, {{}, {{shape_params.data_shape}}}}); @@ -227,7 +238,8 @@ void GatherWeightsDecompression::SetUp() { output_precision, decompression_sub, reshape_on_decompression, - per_tensor_zp); + per_tensor_zp, + per_tensor_scale); if (output_precision == ov::element::f16) { abs_threshold = 1.0f;