natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix regression model orca (#24100) 

### Details:
 - *Fix regression for model orca.*
 - *Add test to conver orca for CPU/GPU test.*

### Tickets:
 - *138406*

---------

Signed-off-by: xipingya <xiping.yan@intel.com>
This commit is contained in:
Xiping Yan 2024-05-17 09:49:23 +08:00 committed by GitHub
parent 61822be8ff
commit fc5e60c77e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 113 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
src/plugins/intel_cpu/src/nodes/gather.cpp
View File

@ -171,6 +171,7 @@ void Gather::initSupportedPrimitiveDescriptors() {
}
scale_group_size =
getInputShapeAtPort(GATHER_DATA).getElementsCount() / getInputShapeAtPort(GATHER_SCALE).getElementsCount();
have_scalar_scale = getInputShapeAtPort(GATHER_SCALE).getElementsCount() == 1u;
if (getOriginalInputsNumber() == 5u) {
ov::element::Type zpPrecision = getOriginalInputPrecisionAtPort(GATHER_ZP);
@ -179,6 +180,7 @@ void Gather::initSupportedPrimitiveDescriptors() {
}
have_zp = true;
have_scalar_zp = getInputShapeAtPort(GATHER_ZP).getElementsCount() == 1u;
zp_group_size =
getInputShapeAtPort(GATHER_DATA).getElementsCount() / getInputShapeAtPort(GATHER_ZP).getElementsCount();
addSupportedPrimDesc({{LayoutType::ncsp, dataPrecision},
@ -630,7 +632,6 @@ void Gather::execCompressed4Bit() {
const auto* scale = getSrcDataAtPortAs<float_t>(GATHER_SCALE);
const size_t dstAfterBatchSize = betweenBatchAndAxisSize * specIdxAndAfterAxSize;
parallel_for2d(beforeBatchSize, specIndicesSize, [&](const size_t b, const size_t j) {
int ii = srcIndices[b * specIndicesSize + j];
if (ii < 0) {
@ -649,15 +650,46 @@ void Gather::execCompressed4Bit() {
OUT_TYPE* pdst = &dstData[dstIdx];
const size_t scale_offset = srcIdx / scale_group_size;
auto cur_zp = have_zp ? zp[srcIdx / zp_group_size] : 0;
size_t p = srcIdx;
size_t dst_idx = 0;
for (; p < srcIdx + afterAxisSize; p++) {
auto val = srcData[p >> 1];
pdst[dst_idx] = static_cast<OUT_TYPE>((get4Bit(val, p % 2) - cur_zp) * scale[scale_offset]);
dst_idx++;
// heuristic:
// ((isAxisInputConst && axis == 0) && (cond1 || cond2)) take >99% probability
bool processed = false;
if (isAxisInputConst && axis == 0) {
bool cond1 = have_zp && zp_group_size == scale_group_size;
bool cond2 = (!have_zp) || have_scalar_zp;
bool cond3 = have_scalar_scale && cond2;
if (cond3) {
processed = true;
for (; p < srcIdx + afterAxisSize; p++) {
auto val = srcData[p >> 1];
pdst[dst_idx] = static_cast<OUT_TYPE>((get4Bit(val, p % 2) - zp[0]) * scale[0]);
dst_idx++;
}
} else if (cond1 || cond2) {
processed = true;
for (; p < srcIdx + afterAxisSize; p += scale_group_size) {
const auto& cur_scale = scale[p / scale_group_size];
const auto& cur_zp = cond2 ? zp[0] : zp[p / zp_group_size];
for (size_t g = p; g < p + scale_group_size; g++) {
auto val = srcData[g >> 1];
pdst[dst_idx] = static_cast<OUT_TYPE>((get4Bit(val, g % 2) - cur_zp) * cur_scale);
dst_idx++;
}
}
}
}
// Reference
if (!processed) {
for (; p < srcIdx + afterAxisSize; p++) {
auto val = srcData[p >> 1];
const size_t scale_offset = p / scale_group_size;
auto cur_zp = have_zp ? zp[p / zp_group_size] : 0;
pdst[dst_idx] = static_cast<OUT_TYPE>((get4Bit(val, p % 2) - cur_zp) * scale[scale_offset]);
dst_idx++;
}
}
}
} else {
@ -699,13 +731,47 @@ void Gather::execCompressed8Bit() {
size_t srcIdx = c1 + axisAndAfterAxisSize * i;
size_t dstIdx = c2 + specIdxAndAfterAxSize * i;
const IN_TYPE* psrc = &srcData[srcIdx];
OUT_TYPE* pdst = &dstData[dstIdx];
const size_t scale_offset = srcIdx / scale_group_size;
auto cur_zp = have_zp ? zp[srcIdx / zp_group_size] : 0;
for (size_t p = 0; p < afterAxisSize; p++) {
pdst[p] = static_cast<OUT_TYPE>((static_cast<float>(psrc[p]) - cur_zp) * scale[scale_offset]);
size_t p = srcIdx;
size_t dst_idx = 0;
// heuristic:
// ((isAxisInputConst && axis == 0) && (cond1 || cond2)) take >99% probability
bool processed = false;
if (isAxisInputConst && axis == 0) {
bool cond1 = have_zp && zp_group_size == scale_group_size;
bool cond2 = (!have_zp) || have_scalar_zp;
bool cond3 = have_scalar_scale && cond2;
if (cond3) {
processed = true;
for (; p < srcIdx + afterAxisSize; p++) {
pdst[dst_idx] = static_cast<OUT_TYPE>((static_cast<float>(srcData[p]) - zp[0]) * scale[0]);
dst_idx++;
}
} else if (cond1 || cond2) {
processed = true;
for (; p < srcIdx + afterAxisSize; p += scale_group_size) {
const auto& cur_scale = scale[p / scale_group_size];
const auto& cur_zp = cond2 ? zp[0] : zp[p / zp_group_size];
for (size_t g = p; g < p + scale_group_size; g++) {
pdst[dst_idx] =
static_cast<OUT_TYPE>((static_cast<float>(srcData[g]) - cur_zp) * cur_scale);
dst_idx++;
}
}
}
}
// Reference
if (!processed) {
for (; p < srcIdx + afterAxisSize; p++) {
const size_t scale_offset = p / scale_group_size;
auto cur_zp = have_zp ? zp[p / zp_group_size] : 0;
pdst[dst_idx] =
static_cast<OUT_TYPE>((static_cast<float>(srcData[p]) - cur_zp) * scale[scale_offset]);
dst_idx++;
}
}
}
} else {

2
src/plugins/intel_cpu/src/nodes/gather.h
View File

@ -106,6 +106,8 @@ private:
static constexpr size_t GATHER_ZP = 4;
bool have_zp = false;
bool have_scalar_zp = false;
bool have_scalar_scale = false;
size_t zp_group_size = 1u;
size_t scale_group_size = 1u;

5
src/plugins/intel_cpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp
View File

@ -24,12 +24,14 @@ const std::vector<ov::element::Type> weights_precisions = {ov::element::u8,
const std::vector<GWDShapeParams> input_shapes_basic = {
{{2, 5}, {{-1, -1}, {{2, 3}}}, 0, 0},
{{15, 32}, {{-1, -1}, {{2, 3}}}, 1, 0, 16},
{{15, 32}, {{-1, -1}, {{2, 3}}}, 0, 0, 16},
{{2, 5}, {{}, {{2, 3}}}, 1, -1},
{{15, 16, 2}, {{-1, -1}, {{2, 3}}}, 0, 0},
};
const std::vector<bool> add_decompression_sub = {true, false};
const std::vector<bool> reshape_on_decompression = {true, false};
const std::vector<bool> per_tensor_zp = {true, false};
const std::vector<bool> per_tensor_scale = {true, false};
INSTANTIATE_TEST_SUITE_P(smoke_GatherCompressedWeights_basic,
GatherWeightsDecompression,
@ -39,7 +41,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_GatherCompressedWeights_basic,
::testing::ValuesIn(output_precisions),
::testing::ValuesIn(add_decompression_sub),
::testing::ValuesIn(reshape_on_decompression),
::testing::ValuesIn(per_tensor_zp)),
::testing::ValuesIn(per_tensor_zp),
::testing::ValuesIn(per_tensor_scale)),
GatherWeightsDecompression::get_test_case_name);
} // namespace

5
src/plugins/intel_gpu/tests/functional/shared_tests_instances/subgraph_tests/gather_weights_decompression.cpp
View File

@ -22,12 +22,14 @@ const std::vector<ov::element::Type> weights_precisions = {ov::element::u8,
const std::vector<GWDShapeParams> input_shapes_basic = {
{{2, 5}, {{-1, -1}, {{2, 3}}}, 1, 1},
{{15, 32}, {{-1, -1}, {{2, 3}}}, 1, 0, 16},
{{15, 32}, {{-1, -1}, {{2, 3}}}, 0, 0, 16},
{{2, 5}, {{}, {{2, 3}}}, 1, -1},
{{15, 16, 2}, {{-1, -1}, {{2, 3}}}, 0, 0},
};
const std::vector<bool> add_decompression_sub = {true, false};
const std::vector<bool> reshape_on_decompression = {true, false};
const std::vector<bool> per_tensor_zp = {true, false};
const std::vector<bool> per_tensor_scale = {true, false};
INSTANTIATE_TEST_SUITE_P(smoke_GatherCompressedWeights_basic,
GatherWeightsDecompression,
@ -37,7 +39,8 @@ INSTANTIATE_TEST_SUITE_P(smoke_GatherCompressedWeights_basic,
::testing::ValuesIn(output_precisions),
::testing::ValuesIn(add_decompression_sub),
::testing::ValuesIn(reshape_on_decompression),
::testing::ValuesIn(per_tensor_zp)),
::testing::ValuesIn(per_tensor_zp),
::testing::ValuesIn(per_tensor_scale)),
GatherWeightsDecompression::get_test_case_name);
} // namespace

7
src/tests/functional/shared_test_classes/include/shared_test_classes/subgraph/gather_weights_decompression.hpp
View File

@ -52,6 +52,7 @@ using GatherWeightsDecompressionParams = std::tuple<std::string, // Devic
ov::element::Type, // output type
bool, // decompression subtract
bool, // reshape on decompression constants
bool, // per-tensor scale
bool>; // per-tensor zero-point
class GatherWeightsDecompression : public testing::WithParamInterface<GatherWeightsDecompressionParams>,
@ -69,14 +70,16 @@ protected:
const ov::element::Type output_precision,
const bool add_subtract,
const bool reshape_on_decompression,
const bool per_tensor_zp);
const bool per_tensor_zp,
const bool per_tensor_scale);
std::shared_ptr<ov::Node> init_compressed_weights_subgraph(const ov::Shape& data_shape,
const int group_size,
const ov::element::Type data_precision,
const ov::element::Type output_precision,
const bool add_subtract,
const bool reshape_on_decompression_constant,
const bool per_tensor_zp);
const bool per_tensor_zp,
const bool per_tensor_scale);
void generate_inputs(const std::vector<ov::Shape>& target_input_static_shapes) override;
void check_results();
void SetUp() override;

28
src/tests/functional/shared_test_classes/src/subgraph/gather_weights_decompression.cpp
View File

@ -17,6 +17,7 @@ std::string GatherWeightsDecompression::get_test_case_name(
bool decompression_sub;
bool reshape_on_decompression;
bool per_tensor_zp;
bool per_tensor_scale;
std::tie(target_device,
shape_params,
@ -24,7 +25,8 @@ std::string GatherWeightsDecompression::get_test_case_name(
output_precision,
decompression_sub,
reshape_on_decompression,
per_tensor_zp) = obj.param;
per_tensor_zp,
per_tensor_scale) = obj.param;
std::ostringstream result;
result << "target_device=" << target_device << "_";
@ -34,12 +36,15 @@ std::string GatherWeightsDecompression::get_test_case_name(
for (const auto& actual_shape : shape_params.indices_shape.second) {
result << ov::test::utils::partialShape2str({actual_shape}) << "_";
}
result << "axis=" << shape_params.axis << "_";
result << "batch_dims=" << shape_params.batch_dims << "_";
result << "group_size=" << shape_params.decompression_group_size << "_";
result << "data_precision=" << data_precision << "_";
result << "output_precision=" << output_precision << "_";
result << "decompression_subtract=" << decompression_sub << "_";
result << "reshape_on_decompression=" << reshape_on_decompression << "_";
result << "per_tensor_zp=" << per_tensor_zp;
result << "per_tensor_scale=" << per_tensor_scale;
return result.str();
}
@ -53,7 +58,8 @@ std::shared_ptr<ov::Model> GatherWeightsDecompression::init_subgraph(const ov::S
const ov::element::Type output_precision,
const bool add_subtract,
const bool reshape_on_decompression,
const bool per_tensor_zp) {
const bool per_tensor_zp,
const bool per_tensor_scale) {
ov::ParameterVector params{std::make_shared<ov::op::v0::Parameter>(ov::element::i32, indices_shape)};
auto axis_const = ov::op::v0::Constant::create(ov::element::i32, {1}, {axis});
const auto data_subgraph = init_compressed_weights_subgraph(data_shape,
@ -62,7 +68,8 @@ std::shared_ptr<ov::Model> GatherWeightsDecompression::init_subgraph(const ov::S
output_precision,
add_subtract,
reshape_on_decompression,
per_tensor_zp);
per_tensor_zp,
per_tensor_scale);
auto gather = std::make_shared<ov::op::v8::Gather>(data_subgraph, params[0], axis_const, batch_dims);
gather->set_friendly_name("gather_node");
@ -75,7 +82,8 @@ std::shared_ptr<ov::Node> GatherWeightsDecompression::init_compressed_weights_su
const ov::element::Type output_precision,
const bool add_subtract,
const bool reshape_on_decompression_constant,
const bool per_tensor_zp) {
const bool per_tensor_zp,
const bool per_tensor_scale) {
const bool group_decompression = group_size != -1;
// Weights has shape [I, D], where
// I - index
@ -141,7 +149,8 @@ std::shared_ptr<ov::Node> GatherWeightsDecompression::init_compressed_weights_su
in_data.start_from = -0.5;
in_data.range = 1;
in_data.resolution = 30000;
auto scale_tensor = ov::test::utils::create_and_fill_tensor(output_precision, scaleshift_const_shape, in_data);
auto shift_tensor_shape = per_tensor_scale ? ov::Shape{1} : scaleshift_const_shape;
auto scale_tensor = ov::test::utils::create_and_fill_tensor(output_precision, shift_tensor_shape, in_data);
for (size_t i = 0; i < scale_tensor.get_size(); i++) {
if (output_precision == ov::element::f16)
scale_tensor.data<ov::float16>()[i] /= ov::float16(16.f);
@ -149,7 +158,7 @@ std::shared_ptr<ov::Node> GatherWeightsDecompression::init_compressed_weights_su
scale_tensor.data<float>()[i] /= 16.f;
}
std::shared_ptr<ov::Node> scale_const = std::make_shared<ov::op::v0::Constant>(scale_tensor);
if (reshape_on_decompression_constant) {
if (reshape_on_decompression_constant && !per_tensor_scale) {
auto scale_reshape_const =
ov::op::v0::Constant::create(ov::element::i32, {scaleshift_target_shape.size()}, scaleshift_target_shape);
auto scale_reshape = std::make_shared<ov::op::v1::Reshape>(scale_const, scale_reshape_const, false);
@ -204,6 +213,7 @@ void GatherWeightsDecompression::SetUp() {
bool decompression_sub;
bool reshape_on_decompression;
bool per_tensor_zp;
bool per_tensor_scale;
std::tie(targetDevice,
shape_params,
@ -211,7 +221,8 @@ void GatherWeightsDecompression::SetUp() {
output_precision,
decompression_sub,
reshape_on_decompression,
per_tensor_zp) = GetParam();
per_tensor_zp,
per_tensor_scale) = GetParam();
init_input_shapes({shape_params.indices_shape, {{}, {{shape_params.data_shape}}}});
@ -227,7 +238,8 @@ void GatherWeightsDecompression::SetUp() {
output_precision,
decompression_sub,
reshape_on_decompression,
per_tensor_zp);
per_tensor_zp,
per_tensor_scale);
if (output_precision == ov::element::f16) {
abs_threshold = 1.0f;